From 8630bb0ad40e487bb10a38e701f50b47b9694365 Mon Sep 17 00:00:00 2001 From: cketti Date: Thu, 13 Nov 2014 17:09:38 +0100 Subject: [PATCH 01/10] Add simple test to check if writing a parsed message leads to input data --- .../fsck/k9/mail/ReconstructMessageTest.java | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 tests/src/com/fsck/k9/mail/ReconstructMessageTest.java diff --git a/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java b/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java new file mode 100644 index 000000000..45f7c8f5c --- /dev/null +++ b/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java @@ -0,0 +1,67 @@ +package com.fsck.k9.mail; + + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.InputStream; + +import android.test.AndroidTestCase; + +import com.fsck.k9.mail.internet.BinaryTempFileBody; +import com.fsck.k9.mail.internet.MimeMessage; + + +public class ReconstructMessageTest extends AndroidTestCase { + + public void testMessage() throws IOException, MessagingException { + String messageSource = + "From: from@example.com\r\n" + + "To: to@example.com\r\n" + + "Subject: Test Message \r\n" + + "Date: Thu, 13 Nov 2014 17:09:38 +0100\r\n" + + "MIME-Version: 1.0\r\n" + + "Content-Type: multipart/mixed;\r\n" + + " boundary=\"----Boundary\"\r\n" + + "Content-Transfer-Encoding: 8bit\r\n" + + "\r\n" + + "This is a multipart MIME message.\r\n" + + "------Boundary\r\n" + + "Content-Type: text/plain; charset=utf-8\r\n" + + "Content-Transfer-Encoding: 8bit\r\n" + + "\r\n" + + "Testing.\r\n" + + "This is a text body with some greek characters.\r\n" + + "αβγδεζηθ\r\n" + + "End of test.\r\n" + + "\r\n" + + "------Boundary\r\n" + + "Content-Type: text/plain\r\n" + + "Content-Transfer-Encoding: quoted-printable\r\n" + + "\r\n" + + "=2E=2E=2E\r\n" + + "\r\n" + + "------Boundary--\r\n"; + + BinaryTempFileBody.setTempDirectory(getContext().getCacheDir()); + + InputStream messageInputStream = new ByteArrayInputStream(messageSource.getBytes()); + MimeMessage message; + try { + message = new MimeMessage(messageInputStream, true); + } finally { + messageInputStream.close(); + } + + ByteArrayOutputStream messageOutputStream = new ByteArrayOutputStream(); + try { + message.writeTo(messageOutputStream); + } finally { + messageOutputStream.close(); + } + + String reconstructedMessage = new String(messageOutputStream.toByteArray()); + + assertEquals(messageSource, reconstructedMessage); + } +} From bcb6c75c2e87854f6b5479d21d6d3a7c3e2a9c4f Mon Sep 17 00:00:00 2001 From: cketti Date: Thu, 13 Nov 2014 22:40:58 +0100 Subject: [PATCH 02/10] Add support for storing raw header fields --- src/com/fsck/k9/mail/Message.java | 3 + src/com/fsck/k9/mail/Part.java | 2 + .../fsck/k9/mail/internet/MimeBodyPart.java | 5 + src/com/fsck/k9/mail/internet/MimeHeader.java | 110 ++++++++++++++---- .../fsck/k9/mail/internet/MimeMessage.java | 9 +- .../k9/mail/store/local/LocalMessage.java | 5 + 6 files changed, 109 insertions(+), 25 deletions(-) diff --git a/src/com/fsck/k9/mail/Message.java b/src/com/fsck/k9/mail/Message.java index 5a4d7a34a..3735f1f58 100644 --- a/src/com/fsck/k9/mail/Message.java +++ b/src/com/fsck/k9/mail/Message.java @@ -135,6 +135,9 @@ public abstract class Message implements Part, CompositeBody { @Override public abstract void addHeader(String name, String value) throws MessagingException; + @Override + public abstract void addRawHeader(String name, String raw) throws MessagingException; + @Override public abstract void setHeader(String name, String value) throws MessagingException; diff --git a/src/com/fsck/k9/mail/Part.java b/src/com/fsck/k9/mail/Part.java index 38c1ad9da..d7970d433 100644 --- a/src/com/fsck/k9/mail/Part.java +++ b/src/com/fsck/k9/mail/Part.java @@ -7,6 +7,8 @@ import java.io.OutputStream; public interface Part { public void addHeader(String name, String value) throws MessagingException; + public void addRawHeader(String name, String raw) throws MessagingException; + public void removeHeader(String name) throws MessagingException; public void setHeader(String name, String value) throws MessagingException; diff --git a/src/com/fsck/k9/mail/internet/MimeBodyPart.java b/src/com/fsck/k9/mail/internet/MimeBodyPart.java index c4bca428a..a542e47fc 100644 --- a/src/com/fsck/k9/mail/internet/MimeBodyPart.java +++ b/src/com/fsck/k9/mail/internet/MimeBodyPart.java @@ -47,6 +47,11 @@ public class MimeBodyPart extends BodyPart { mHeader.addHeader(name, value); } + @Override + public void addRawHeader(String name, String raw) { + mHeader.addRawHeader(name, raw); + } + @Override public void setHeader(String name, String value) { mHeader.setHeader(name, value); diff --git a/src/com/fsck/k9/mail/internet/MimeHeader.java b/src/com/fsck/k9/mail/internet/MimeHeader.java index 12bd5fd31..f7313a4f2 100644 --- a/src/com/fsck/k9/mail/internet/MimeHeader.java +++ b/src/com/fsck/k9/mail/internet/MimeHeader.java @@ -52,7 +52,13 @@ public class MimeHeader { } public void addHeader(String name, String value) { - mFields.add(new Field(name, MimeUtility.foldAndEncode(value))); + Field field = Field.newNameValueField(name, MimeUtility.foldAndEncode(value)); + mFields.add(field); + } + + void addRawHeader(String name, String raw) { + Field field = Field.newRawField(name, raw); + mFields.add(field); } public void setHeader(String name, String value) { @@ -66,7 +72,7 @@ public class MimeHeader { public Set getHeaderNames() { Set names = new LinkedHashSet(); for (Field field : mFields) { - names.add(field.name); + names.add(field.getName()); } return names; } @@ -74,8 +80,8 @@ public class MimeHeader { public String[] getHeader(String name) { List values = new ArrayList(); for (Field field : mFields) { - if (field.name.equalsIgnoreCase(name)) { - values.add(field.value); + if (field.getName().equalsIgnoreCase(name)) { + values.add(field.getValue()); } } if (values.isEmpty()) { @@ -87,7 +93,7 @@ public class MimeHeader { public void removeHeader(String name) { List removeFields = new ArrayList(); for (Field field : mFields) { - if (field.name.equalsIgnoreCase(name)) { + if (field.getName().equalsIgnoreCase(name)) { removeFields.add(field); } } @@ -97,27 +103,35 @@ public class MimeHeader { public void writeTo(OutputStream out) throws IOException { BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out), 1024); for (Field field : mFields) { - if (!Utility.arrayContains(writeOmitFields, field.name)) { - String v = field.value; - - if (hasToBeEncoded(v)) { - Charset charset = null; - - if (mCharset != null) { - charset = Charset.forName(mCharset); - } - v = EncoderUtil.encodeEncodedWord(field.value, charset); + if (!Utility.arrayContains(writeOmitFields, field.getName())) { + if (field.hasRawData()) { + writer.write(field.getRaw()); + } else { + writeNameValueField(writer, field); } - - writer.write(field.name); - writer.write(": "); - writer.write(v); writer.write("\r\n"); } } writer.flush(); } + private void writeNameValueField(BufferedWriter writer, Field field) throws IOException { + String value = field.getValue(); + + if (hasToBeEncoded(value)) { + Charset charset = null; + + if (mCharset != null) { + charset = Charset.forName(mCharset); + } + value = EncoderUtil.encodeEncodedWord(field.getValue(), charset); + } + + writer.write(field.getName()); + writer.write(": "); + writer.write(value); + } + // encode non printable characters except LF/CR/TAB codes. private boolean hasToBeEncoded(String text) { for (int i = 0; i < text.length(); i++) { @@ -133,19 +147,67 @@ public class MimeHeader { private static class Field { private final String name; - private final String value; + private final String raw; + + public static Field newNameValueField(String name, String value) { + if (value == null) { + throw new NullPointerException("Argument 'value' cannot be null"); + } + + return new Field(name, value, null); + } + + public static Field newRawField(String name, String raw) { + if (raw == null) { + throw new NullPointerException("Argument 'raw' cannot be null"); + } + if (name != null && !raw.startsWith(name + ":")) { + throw new IllegalArgumentException("The value of 'raw' needs to start with the supplied field name " + + "followed by a colon"); + } + + return new Field(name, null, raw); + } + + private Field(String name, String value, String raw) { + if (name == null) { + throw new NullPointerException("Argument 'name' cannot be null"); + } - public Field(String name, String value) { this.name = name; this.value = value; + this.raw = raw; + } + + public String getName() { + return name; + } + + public String getValue() { + if (value != null) { + return value; + } + + int delimiterIndex = raw.indexOf(':'); + if (delimiterIndex == raw.length() - 1) { + return ""; + } + + return raw.substring(delimiterIndex + 1).trim(); + } + + public String getRaw() { + return raw; + } + + public boolean hasRawData() { + return raw != null; } @Override public String toString() { - StringBuilder sb = new StringBuilder("("); - sb.append(name).append('=').append(value).append(')'); - return sb.toString(); + return (hasRawData()) ? getRaw() : getName() + ": " + getValue(); } } diff --git a/src/com/fsck/k9/mail/internet/MimeMessage.java b/src/com/fsck/k9/mail/internet/MimeMessage.java index 75fa217d8..c870fda35 100644 --- a/src/com/fsck/k9/mail/internet/MimeMessage.java +++ b/src/com/fsck/k9/mail/internet/MimeMessage.java @@ -423,6 +423,11 @@ public class MimeMessage extends Message { mHeader.addHeader(name, value); } + @Override + public void addRawHeader(String name, String raw) { + mHeader.addRawHeader(name, raw); + } + @Override public void setHeader(String name, String value) throws UnavailableStorageException { mHeader.setHeader(name, value); @@ -598,7 +603,9 @@ public class MimeMessage extends Message { public void field(Field parsedField) throws MimeException { expect(Part.class); try { - ((Part)stack.peek()).addHeader(parsedField.getName(), parsedField.getBody().trim()); + String name = parsedField.getName(); + String raw = parsedField.getRaw().toString(); + ((Part) stack.peek()).addRawHeader(name, raw); } catch (MessagingException me) { throw new Error(me); } diff --git a/src/com/fsck/k9/mail/store/local/LocalMessage.java b/src/com/fsck/k9/mail/store/local/LocalMessage.java index 474a4c849..ba3274b54 100644 --- a/src/com/fsck/k9/mail/store/local/LocalMessage.java +++ b/src/com/fsck/k9/mail/store/local/LocalMessage.java @@ -507,6 +507,11 @@ public class LocalMessage extends MimeMessage { super.addHeader(name, value); } + @Override + public void addRawHeader(String name, String raw) { + throw new RuntimeException("Not supported"); + } + @Override public void setHeader(String name, String value) throws UnavailableStorageException { if (!mHeadersLoaded) From 2404b80b04feb8e1cd04fd636a22d2a4d96e5c5b Mon Sep 17 00:00:00 2001 From: cketti Date: Thu, 13 Nov 2014 23:24:27 +0100 Subject: [PATCH 03/10] Fix MessageTest now that we preserve line breaks in headers --- tests/src/com/fsck/k9/mail/MessageTest.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/src/com/fsck/k9/mail/MessageTest.java b/tests/src/com/fsck/k9/mail/MessageTest.java index 3e56b228d..8a7e135b8 100644 --- a/tests/src/com/fsck/k9/mail/MessageTest.java +++ b/tests/src/com/fsck/k9/mail/MessageTest.java @@ -191,7 +191,8 @@ public class MessageTest extends AndroidTestCase { + "Content-Transfer-Encoding: 7bit\r\n" + "\r\n" + "------Boundary102\r\n" - + "Content-Type: text/plain; charset=utf-8\r\n" + + "Content-Type: text/plain;\r\n" + + " charset=utf-8\r\n" + "Content-Transfer-Encoding: quoted-printable\r\n" + "\r\n" + "Testing=2E\r\n" @@ -200,7 +201,8 @@ public class MessageTest extends AndroidTestCase { + "End of test=2E\r\n" + "\r\n" + "------Boundary102\r\n" - + "Content-Type: text/plain; charset=utf-8\r\n" + + "Content-Type: text/plain;\r\n" + + " charset=utf-8\r\n" + "Content-Transfer-Encoding: quoted-printable\r\n" + "\r\n" + "Testing=2E\r\n" @@ -228,7 +230,8 @@ public class MessageTest extends AndroidTestCase { + "Content-Transfer-Encoding: 7bit\r\n" + "\r\n" + "------Boundary101\r\n" - + "Content-Type: text/plain; charset=utf-8\r\n" + + "Content-Type: text/plain;\r\n" + + " charset=utf-8\r\n" + "Content-Transfer-Encoding: quoted-printable\r\n" + "\r\n" + "Testing=2E\r\n" @@ -237,7 +240,8 @@ public class MessageTest extends AndroidTestCase { + "End of test=2E\r\n" + "\r\n" + "------Boundary101\r\n" - + "Content-Type: text/plain; charset=utf-8\r\n" + + "Content-Type: text/plain;\r\n" + + " charset=utf-8\r\n" + "Content-Transfer-Encoding: quoted-printable\r\n" + "\r\n" + "Testing=2E\r\n" From 51a60b5ad33ed381b3851bb6c87228000015d235 Mon Sep 17 00:00:00 2001 From: cketti Date: Fri, 14 Nov 2014 00:27:04 +0100 Subject: [PATCH 04/10] Modify ReconstructMessageTest to highlight more problems --- tests/src/com/fsck/k9/mail/ReconstructMessageTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java b/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java index 45f7c8f5c..0989581cf 100644 --- a/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java +++ b/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java @@ -20,10 +20,10 @@ public class ReconstructMessageTest extends AndroidTestCase { "To: to@example.com\r\n" + "Subject: Test Message \r\n" + "Date: Thu, 13 Nov 2014 17:09:38 +0100\r\n" + - "MIME-Version: 1.0\r\n" + "Content-Type: multipart/mixed;\r\n" + " boundary=\"----Boundary\"\r\n" + "Content-Transfer-Encoding: 8bit\r\n" + + "MIME-Version: 1.0\r\n" + "\r\n" + "This is a multipart MIME message.\r\n" + "------Boundary\r\n" + From d32d6eed0e3d8a7dbd91df156872512450fbcb61 Mon Sep 17 00:00:00 2001 From: cketti Date: Fri, 14 Nov 2014 00:58:43 +0100 Subject: [PATCH 05/10] Move "magic" from Part.setBody() implementations to MimeMessageHelper.setBody() Now adding message bodies during parsing won't set/modify headers. --- src/com/fsck/k9/activity/MessageCompose.java | 9 ++-- .../k9/controller/MessagingController.java | 5 +- .../fsck/k9/mail/internet/MimeBodyPart.java | 21 +------- .../fsck/k9/mail/internet/MimeMessage.java | 16 ------ .../k9/mail/internet/MimeMessageHelper.java | 52 +++++++++++++++++++ .../fsck/k9/mail/internet/MimeUtility.java | 2 +- src/com/fsck/k9/mail/store/ImapStore.java | 5 +- .../fsck/k9/mail/store/local/LocalFolder.java | 17 +++--- tests/src/com/fsck/k9/mail/MessageTest.java | 4 +- .../fsck/k9/mail/internet/ViewablesTest.java | 10 ++-- 10 files changed, 83 insertions(+), 58 deletions(-) create mode 100644 src/com/fsck/k9/mail/internet/MimeMessageHelper.java diff --git a/src/com/fsck/k9/activity/MessageCompose.java b/src/com/fsck/k9/activity/MessageCompose.java index ebf53391c..31d3a4cd7 100644 --- a/src/com/fsck/k9/activity/MessageCompose.java +++ b/src/com/fsck/k9/activity/MessageCompose.java @@ -104,6 +104,7 @@ import com.fsck.k9.mail.Part; import com.fsck.k9.mail.internet.MimeBodyPart; import com.fsck.k9.mail.internet.MimeHeader; import com.fsck.k9.mail.internet.MimeMessage; +import com.fsck.k9.mail.internet.MimeMessageHelper; import com.fsck.k9.mail.internet.MimeMultipart; import com.fsck.k9.mail.internet.MimeUtility; import com.fsck.k9.mail.internet.TextBody; @@ -1404,10 +1405,10 @@ public class MessageCompose extends K9Activity implements OnClickListener, MimeMultipart mp = new MimeMultipart(); mp.addBodyPart(new MimeBodyPart(composedMimeMessage)); addAttachmentsToMessage(mp); - message.setBody(mp); + MimeMessageHelper.setBody(message, mp); } else { // If no attachments, our multipart/alternative part is the only one we need. - message.setBody(composedMimeMessage); + MimeMessageHelper.setBody(message, composedMimeMessage); } } else if (mMessageFormat == SimpleMessageFormat.TEXT) { // Text-only message. @@ -1415,10 +1416,10 @@ public class MessageCompose extends K9Activity implements OnClickListener, MimeMultipart mp = new MimeMultipart(); mp.addBodyPart(new MimeBodyPart(body, "text/plain")); addAttachmentsToMessage(mp); - message.setBody(mp); + MimeMessageHelper.setBody(message, mp); } else { // No attachments to include, just stick the text body in the message and call it good. - message.setBody(body); + MimeMessageHelper.setBody(message, body); } } diff --git a/src/com/fsck/k9/controller/MessagingController.java b/src/com/fsck/k9/controller/MessagingController.java index 07f2f5e05..c58c97e1b 100644 --- a/src/com/fsck/k9/controller/MessagingController.java +++ b/src/com/fsck/k9/controller/MessagingController.java @@ -79,6 +79,7 @@ import com.fsck.k9.mail.Pusher; import com.fsck.k9.mail.Store; import com.fsck.k9.mail.Transport; import com.fsck.k9.mail.internet.MimeMessage; +import com.fsck.k9.mail.internet.MimeMessageHelper; import com.fsck.k9.mail.internet.MimeUtility; import com.fsck.k9.mail.internet.TextBody; import com.fsck.k9.mail.store.local.LocalFolder; @@ -2712,7 +2713,7 @@ public class MessagingController implements Runnable { LocalFolder localFolder = (LocalFolder)localStore.getFolder(account.getErrorFolderName()); MimeMessage message = new MimeMessage(); - message.setBody(new TextBody(body)); + MimeMessageHelper.setBody(message, new TextBody(body)); message.setFlag(Flag.X_DOWNLOADED_FULL, true); message.setSubject(subject); @@ -3205,7 +3206,7 @@ public class MessagingController implements Runnable { //FIXME: This is an ugly hack that won't be needed once the Message objects have been united. Message remoteMessage = remoteFolder.getMessage(message.getUid()); - remoteMessage.setBody(message.getBody()); + MimeMessageHelper.setBody(remoteMessage, message.getBody()); remoteFolder.fetchPart(remoteMessage, part, null); localFolder.updateMessage((LocalMessage)message); diff --git a/src/com/fsck/k9/mail/internet/MimeBodyPart.java b/src/com/fsck/k9/mail/internet/MimeBodyPart.java index a542e47fc..32e3e2654 100644 --- a/src/com/fsck/k9/mail/internet/MimeBodyPart.java +++ b/src/com/fsck/k9/mail/internet/MimeBodyPart.java @@ -35,7 +35,7 @@ public class MimeBodyPart extends BodyPart { if (mimeType != null) { addHeader(MimeHeader.HEADER_CONTENT_TYPE, mimeType); } - setBody(body); + MimeMessageHelper.setBody(this, body); } private String getFirstHeader(String name) { @@ -75,25 +75,6 @@ public class MimeBodyPart extends BodyPart { @Override public void setBody(Body body) throws MessagingException { this.mBody = body; - if (body instanceof Multipart) { - Multipart multipart = ((Multipart)body); - multipart.setParent(this); - String type = multipart.getContentType(); - setHeader(MimeHeader.HEADER_CONTENT_TYPE, type); - if ("multipart/signed".equalsIgnoreCase(type)) { - setEncoding(MimeUtil.ENC_7BIT); - } else { - setEncoding(MimeUtil.ENC_8BIT); - } - } else if (body instanceof TextBody) { - String contentType = String.format("%s;\r\n charset=utf-8", getMimeType()); - String name = MimeUtility.getHeaderParameter(getContentType(), "name"); - if (name != null) { - contentType += String.format(";\r\n name=\"%s\"", name); - } - setHeader(MimeHeader.HEADER_CONTENT_TYPE, contentType); - setEncoding(MimeUtil.ENC_8BIT); - } } @Override diff --git a/src/com/fsck/k9/mail/internet/MimeMessage.java b/src/com/fsck/k9/mail/internet/MimeMessage.java index c870fda35..012f2c377 100644 --- a/src/com/fsck/k9/mail/internet/MimeMessage.java +++ b/src/com/fsck/k9/mail/internet/MimeMessage.java @@ -396,22 +396,6 @@ public class MimeMessage extends Message { @Override public void setBody(Body body) throws MessagingException { this.mBody = body; - setHeader("MIME-Version", "1.0"); - if (body instanceof Multipart) { - Multipart multipart = ((Multipart)body); - multipart.setParent(this); - String type = multipart.getContentType(); - setHeader(MimeHeader.HEADER_CONTENT_TYPE, type); - if ("multipart/signed".equalsIgnoreCase(type)) { - setEncoding(MimeUtil.ENC_7BIT); - } else { - setEncoding(MimeUtil.ENC_8BIT); - } - } else if (body instanceof TextBody) { - setHeader(MimeHeader.HEADER_CONTENT_TYPE, String.format("%s;\r\n charset=utf-8", - getMimeType())); - setEncoding(MimeUtil.ENC_8BIT); - } } private String getFirstHeader(String name) { diff --git a/src/com/fsck/k9/mail/internet/MimeMessageHelper.java b/src/com/fsck/k9/mail/internet/MimeMessageHelper.java new file mode 100644 index 000000000..bc1695607 --- /dev/null +++ b/src/com/fsck/k9/mail/internet/MimeMessageHelper.java @@ -0,0 +1,52 @@ +package com.fsck.k9.mail.internet; + + +import com.fsck.k9.mail.Body; +import com.fsck.k9.mail.Message; +import com.fsck.k9.mail.MessagingException; +import com.fsck.k9.mail.Multipart; +import com.fsck.k9.mail.Part; +import org.apache.james.mime4j.util.MimeUtil; + + +public class MimeMessageHelper { + private MimeMessageHelper() { + } + + public static void setBody(Part part, Body body) throws MessagingException { + part.setBody(body); + + if (part instanceof Message) { + part.setHeader("MIME-Version", "1.0"); + } + + if (body instanceof Multipart) { + Multipart multipart = ((Multipart) body); + multipart.setParent(part); + String type = multipart.getContentType(); + part.setHeader(MimeHeader.HEADER_CONTENT_TYPE, type); + if ("multipart/signed".equalsIgnoreCase(type)) { + setEncoding(part, MimeUtil.ENC_7BIT); + } else { + setEncoding(part, MimeUtil.ENC_8BIT); + } + } else if (body instanceof TextBody) { + String contentType = String.format("%s;\r\n charset=utf-8", part.getMimeType()); + String name = MimeUtility.getHeaderParameter(part.getContentType(), "name"); + if (name != null) { + contentType += String.format(";\r\n name=\"%s\"", name); + } + part.setHeader(MimeHeader.HEADER_CONTENT_TYPE, contentType); + + setEncoding(part, MimeUtil.ENC_8BIT); + } + } + + public static void setEncoding(Part part, String encoding) throws MessagingException { + Body body = part.getBody(); + if (body != null) { + body.setEncoding(encoding); + } + part.setHeader(MimeHeader.HEADER_CONTENT_TRANSFER_ENCODING, encoding); + } +} diff --git a/src/com/fsck/k9/mail/internet/MimeUtility.java b/src/com/fsck/k9/mail/internet/MimeUtility.java index 21dad28c7..80efdf5c6 100644 --- a/src/com/fsck/k9/mail/internet/MimeUtility.java +++ b/src/com/fsck/k9/mail/internet/MimeUtility.java @@ -1067,7 +1067,7 @@ public class MimeUtility { String text = readToString(in, charset); // Replace the body with a TextBody that already contains the decoded text - part.setBody(new TextBody(text)); + MimeMessageHelper.setBody(part, new TextBody(text)); return text; } finally { diff --git a/src/com/fsck/k9/mail/store/ImapStore.java b/src/com/fsck/k9/mail/store/ImapStore.java index 9042f9d9b..91406c838 100644 --- a/src/com/fsck/k9/mail/store/ImapStore.java +++ b/src/com/fsck/k9/mail/store/ImapStore.java @@ -48,6 +48,7 @@ import java.util.regex.Pattern; import java.util.zip.Inflater; import java.util.zip.InflaterInputStream; +import com.fsck.k9.mail.internet.MimeMessageHelper; import javax.net.ssl.SSLException; import org.apache.commons.io.IOUtils; @@ -1615,7 +1616,7 @@ public class ImapStore extends Store { if (literal != null) { if (literal instanceof Body) { // Most of the work was done in FetchAttchmentCallback.foundLiteral() - part.setBody((Body)literal); + MimeMessageHelper.setBody(part, (Body) literal); } else if (literal instanceof String) { String bodyString = (String)literal; InputStream bodyStream = new ByteArrayInputStream(bodyString.getBytes()); @@ -1624,7 +1625,7 @@ public class ImapStore extends Store { .getHeader(MimeHeader.HEADER_CONTENT_TRANSFER_ENCODING)[0]; String contentType = part .getHeader(MimeHeader.HEADER_CONTENT_TYPE)[0]; - part.setBody(MimeUtility.decodeBody(bodyStream, + MimeMessageHelper.setBody(part, MimeUtility.decodeBody(bodyStream, contentTransferEncoding, contentType)); } else { // This shouldn't happen diff --git a/src/com/fsck/k9/mail/store/local/LocalFolder.java b/src/com/fsck/k9/mail/store/local/LocalFolder.java index bc5895f04..e84098183 100644 --- a/src/com/fsck/k9/mail/store/local/LocalFolder.java +++ b/src/com/fsck/k9/mail/store/local/LocalFolder.java @@ -19,6 +19,7 @@ import java.util.Set; import java.util.UUID; import java.util.regex.Pattern; +import com.fsck.k9.mail.internet.MimeMessageHelper; import org.apache.commons.io.IOUtils; import org.apache.james.mime4j.util.MimeUtil; @@ -774,17 +775,17 @@ public class LocalFolder extends Folder implements Serializable { // triggering T_MIME_NO_TEXT and T_TVD_MIME_NO_HEADERS // SpamAssassin rules. localMessage.setHeader(MimeHeader.HEADER_CONTENT_TYPE, "text/plain"); - localMessage.setBody(new TextBody("")); + MimeMessageHelper.setBody(localMessage, new TextBody("")); } else if (mp.getCount() == 1 && (mp.getBodyPart(0) instanceof LocalAttachmentBodyPart) == false) { // If we have only one part, drop the MimeMultipart container. BodyPart part = mp.getBodyPart(0); localMessage.setHeader(MimeHeader.HEADER_CONTENT_TYPE, part.getContentType()); - localMessage.setBody(part.getBody()); + MimeMessageHelper.setBody(localMessage, part.getBody()); } else { // Otherwise, attach the MimeMultipart to the message. - localMessage.setBody(mp); + MimeMessageHelper.setBody(localMessage, mp); } } } @@ -1642,11 +1643,13 @@ public class LocalFolder extends Folder implements Serializable { mAccount, attachmentId); if (MimeUtil.isMessage(attachment.getMimeType())) { - attachment.setBody(new LocalAttachmentMessageBody( - contentUri, LocalFolder.this.localStore.mApplication)); + LocalAttachmentMessageBody body = new LocalAttachmentMessageBody( + contentUri, LocalFolder.this.localStore.mApplication); + MimeMessageHelper.setBody(attachment, body); } else { - attachment.setBody(new LocalAttachmentBody( - contentUri, LocalFolder.this.localStore.mApplication)); + LocalAttachmentBody body = new LocalAttachmentBody( + contentUri, LocalFolder.this.localStore.mApplication); + MimeMessageHelper.setBody(attachment, body); } ContentValues cv = new ContentValues(); cv.put("content_uri", contentUri != null ? contentUri.toString() : null); diff --git a/tests/src/com/fsck/k9/mail/MessageTest.java b/tests/src/com/fsck/k9/mail/MessageTest.java index 8a7e135b8..054a3d01a 100644 --- a/tests/src/com/fsck/k9/mail/MessageTest.java +++ b/tests/src/com/fsck/k9/mail/MessageTest.java @@ -5,6 +5,8 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; + +import com.fsck.k9.mail.internet.MimeMessageHelper; import org.apache.commons.io.IOUtils; import org.apache.james.mime4j.codec.Base64InputStream; import org.apache.james.mime4j.util.MimeUtil; @@ -322,7 +324,7 @@ public class MessageTest extends AndroidTestCase { multipartBody.addBodyPart(textBodyPart(MimeUtil.ENC_8BIT)); multipartBody.addBodyPart(textBodyPart(MimeUtil.ENC_QUOTED_PRINTABLE)); multipartBody.addBodyPart(binaryBodyPart()); - message.setBody(multipartBody); + MimeMessageHelper.setBody(message, multipartBody); return message; } diff --git a/tests/src/com/fsck/k9/mail/internet/ViewablesTest.java b/tests/src/com/fsck/k9/mail/internet/ViewablesTest.java index cceb4bff5..f242eb0b9 100644 --- a/tests/src/com/fsck/k9/mail/internet/ViewablesTest.java +++ b/tests/src/com/fsck/k9/mail/internet/ViewablesTest.java @@ -21,7 +21,7 @@ public class ViewablesTest extends AndroidTestCase { // Create message MimeMessage message = new MimeMessage(); - message.setBody(body); + MimeMessageHelper.setBody(message, body); // Extract text ViewableContainer container = MimeUtility.extractTextAndAttachments(getContext(), message); @@ -45,7 +45,7 @@ public class ViewablesTest extends AndroidTestCase { // Create message MimeMessage message = new MimeMessage(); message.setHeader("Content-Type", "text/html"); - message.setBody(body); + MimeMessageHelper.setBody(message, body); // Extract text ViewableContainer container = MimeUtility.extractTextAndAttachments(getContext(), message); @@ -75,7 +75,7 @@ public class ViewablesTest extends AndroidTestCase { // Create message MimeMessage message = new MimeMessage(); - message.setBody(multipart); + MimeMessageHelper.setBody(message, multipart); // Extract text ViewableContainer container = MimeUtility.extractTextAndAttachments(getContext(), message); @@ -119,7 +119,7 @@ public class ViewablesTest extends AndroidTestCase { innerMessage.setRecipients(RecipientType.TO, new Address[] { new Address("to@example.com") }); innerMessage.setSubject("Subject"); innerMessage.setFrom(new Address("from@example.com")); - innerMessage.setBody(innerBody); + MimeMessageHelper.setBody(innerMessage, innerBody); // Create multipart/mixed part MimeMultipart multipart = new MimeMultipart(); @@ -131,7 +131,7 @@ public class ViewablesTest extends AndroidTestCase { // Create message MimeMessage message = new MimeMessage(); - message.setBody(multipart); + MimeMessageHelper.setBody(message, multipart); // Extract text ViewableContainer container = MimeUtility.extractTextAndAttachments(getContext(), message); From 9f4f0cf6a8705105eea1c3a0da4e36f5841dd7c1 Mon Sep 17 00:00:00 2001 From: cketti Date: Tue, 25 Nov 2014 19:59:05 +0100 Subject: [PATCH 06/10] Modify BinaryTempFileBody to retain the encoded body For now this breaks a lot of things, e.g. saving messages to the database and making messages 7-bit safe. --- .../k9/mail/internet/BinaryTempFileBody.java | 25 +++---------------- .../fsck/k9/mail/internet/MimeMessage.java | 3 +-- .../fsck/k9/mail/internet/MimeUtility.java | 21 ++++------------ src/com/fsck/k9/mail/store/ImapStore.java | 6 ++--- tests/src/com/fsck/k9/mail/MessageTest.java | 6 ++--- .../fsck/k9/mail/ReconstructMessageTest.java | 5 ++-- 6 files changed, 17 insertions(+), 49 deletions(-) diff --git a/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java b/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java index f0dd6f473..dceec19e1 100644 --- a/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java +++ b/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java @@ -2,10 +2,7 @@ package com.fsck.k9.mail.internet; import com.fsck.k9.mail.Body; import com.fsck.k9.mail.MessagingException; -import com.fsck.k9.mail.filter.Base64OutputStream; import org.apache.commons.io.IOUtils; -import org.apache.james.mime4j.codec.QuotedPrintableOutputStream; -import org.apache.james.mime4j.util.MimeUtil; import java.io.*; @@ -27,13 +24,12 @@ public class BinaryTempFileBody implements Body { } public void setEncoding(String encoding) throws MessagingException { - mEncoding = encoding; + mEncoding = encoding; } public BinaryTempFileBody() { if (mTempDirectory == null) { - throw new - RuntimeException("setTempDirectory has not been called on BinaryTempFileBody!"); + throw new RuntimeException("setTempDirectory has not been called on BinaryTempFileBody!"); } } @@ -54,22 +50,7 @@ public class BinaryTempFileBody implements Body { public void writeTo(OutputStream out) throws IOException, MessagingException { InputStream in = getInputStream(); try { - boolean closeStream = false; - if (MimeUtil.isBase64Encoding(mEncoding)) { - out = new Base64OutputStream(out); - closeStream = true; - } else if (MimeUtil.isQuotedPrintableEncoded(mEncoding)){ - out = new QuotedPrintableOutputStream(out, false); - closeStream = true; - } - - try { - IOUtils.copy(in, out); - } finally { - if (closeStream) { - out.close(); - } - } + IOUtils.copy(in, out); } finally { in.close(); } diff --git a/src/com/fsck/k9/mail/internet/MimeMessage.java b/src/com/fsck/k9/mail/internet/MimeMessage.java index 012f2c377..8eb3dfe95 100644 --- a/src/com/fsck/k9/mail/internet/MimeMessage.java +++ b/src/com/fsck/k9/mail/internet/MimeMessage.java @@ -531,8 +531,7 @@ public class MimeMessage extends Message { public void body(BodyDescriptor bd, InputStream in) throws IOException { expect(Part.class); try { - Body body = MimeUtility.decodeBody(in, - bd.getTransferEncoding(), bd.getMimeType()); + Body body = MimeUtility.createBody(in, bd.getTransferEncoding(), bd.getMimeType()); ((Part)stack.peek()).setBody(body); } catch (MessagingException me) { throw new Error(me); diff --git a/src/com/fsck/k9/mail/internet/MimeUtility.java b/src/com/fsck/k9/mail/internet/MimeUtility.java index 80efdf5c6..cdd63789e 100644 --- a/src/com/fsck/k9/mail/internet/MimeUtility.java +++ b/src/com/fsck/k9/mail/internet/MimeUtility.java @@ -1127,24 +1127,11 @@ public class MimeUtility { return false; } - /** - * Removes any content transfer encoding from the stream and returns a Body. - * @throws MessagingException - */ - public static Body decodeBody(InputStream in, - String contentTransferEncoding, String contentType) + public static Body createBody(InputStream in, String contentTransferEncoding, String contentType) throws IOException, MessagingException { - /* - * We'll remove any transfer encoding by wrapping the stream. - */ + if (contentTransferEncoding != null) { - contentTransferEncoding = - MimeUtility.getHeaderParameter(contentTransferEncoding, null); - if (MimeUtil.ENC_QUOTED_PRINTABLE.equalsIgnoreCase(contentTransferEncoding)) { - in = new QuotedPrintableInputStream(in); - } else if (MimeUtil.ENC_BASE64.equalsIgnoreCase(contentTransferEncoding)) { - in = new Base64InputStream(in); - } + contentTransferEncoding = MimeUtility.getHeaderParameter(contentTransferEncoding, null); } BinaryTempFileBody tempBody; @@ -1154,12 +1141,14 @@ public class MimeUtility { tempBody = new BinaryTempFileBody(); } tempBody.setEncoding(contentTransferEncoding); + OutputStream out = tempBody.getOutputStream(); try { IOUtils.copy(in, out); } finally { out.close(); } + return tempBody; } diff --git a/src/com/fsck/k9/mail/store/ImapStore.java b/src/com/fsck/k9/mail/store/ImapStore.java index 91406c838..cc714c0e6 100644 --- a/src/com/fsck/k9/mail/store/ImapStore.java +++ b/src/com/fsck/k9/mail/store/ImapStore.java @@ -25,8 +25,6 @@ import java.security.GeneralSecurityException; import java.security.Security; import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; import java.util.Collections; import java.util.Date; import java.util.Deque; @@ -1625,7 +1623,7 @@ public class ImapStore extends Store { .getHeader(MimeHeader.HEADER_CONTENT_TRANSFER_ENCODING)[0]; String contentType = part .getHeader(MimeHeader.HEADER_CONTENT_TYPE)[0]; - MimeMessageHelper.setBody(part, MimeUtility.decodeBody(bodyStream, + MimeMessageHelper.setBody(part, MimeUtility.createBody(bodyStream, contentTransferEncoding, contentType)); } else { // This shouldn't happen @@ -3594,7 +3592,7 @@ public class ImapStore extends Store { String contentType = mPart .getHeader(MimeHeader.HEADER_CONTENT_TYPE)[0]; - return MimeUtility.decodeBody(literal, contentTransferEncoding, + return MimeUtility.createBody(literal, contentTransferEncoding, contentType); } return null; diff --git a/tests/src/com/fsck/k9/mail/MessageTest.java b/tests/src/com/fsck/k9/mail/MessageTest.java index 054a3d01a..c5fc39c74 100644 --- a/tests/src/com/fsck/k9/mail/MessageTest.java +++ b/tests/src/com/fsck/k9/mail/MessageTest.java @@ -332,12 +332,12 @@ public class MessageTest extends AndroidTestCase { private MimeBodyPart binaryBodyPart() throws IOException, MessagingException { String encodedTestString = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" - + "abcdefghijklmnopqrstuvwxyz0123456789+/"; + + "abcdefghijklmnopqrstuvwxyz0123456789+/\r\n"; BinaryTempFileBody tempFileBody = new BinaryTempFileBody(); - InputStream in = new Base64InputStream(new ByteArrayInputStream( - encodedTestString.getBytes("UTF-8"))); + InputStream in = new ByteArrayInputStream( + encodedTestString.getBytes("UTF-8")); OutputStream out = tempFileBody.getOutputStream(); try { diff --git a/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java b/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java index 0989581cf..1e6528c99 100644 --- a/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java +++ b/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java @@ -37,9 +37,10 @@ public class ReconstructMessageTest extends AndroidTestCase { "\r\n" + "------Boundary\r\n" + "Content-Type: text/plain\r\n" + - "Content-Transfer-Encoding: quoted-printable\r\n" + + "Content-Transfer-Encoding: base64\r\n" + "\r\n" + - "=2E=2E=2E\r\n" + + "VGhpcyBpcyBhIHRl\r\n" + + "c3QgbWVzc2FnZQ==\r\n" + "\r\n" + "------Boundary--\r\n"; From f7d3eaa0068c2d03ecdb57742b39b599e85811a7 Mon Sep 17 00:00:00 2001 From: cketti Date: Tue, 25 Nov 2014 23:59:03 +0100 Subject: [PATCH 07/10] Fix setUsing7bitTransport() functionality for BinaryTempFileBody --- .../k9/mail/internet/BinaryTempFileBody.java | 44 ++++++++++++++++++- .../internet/BinaryTempFileMessageBody.java | 4 ++ .../fsck/k9/mail/internet/MimeUtility.java | 5 +-- tests/src/com/fsck/k9/mail/MessageTest.java | 5 +-- 4 files changed, 50 insertions(+), 8 deletions(-) diff --git a/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java b/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java index dceec19e1..677739bbf 100644 --- a/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java +++ b/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java @@ -2,7 +2,10 @@ package com.fsck.k9.mail.internet; import com.fsck.k9.mail.Body; import com.fsck.k9.mail.MessagingException; +import com.fsck.k9.mail.filter.Base64OutputStream; import org.apache.commons.io.IOUtils; +import org.apache.james.mime4j.codec.QuotedPrintableOutputStream; +import org.apache.james.mime4j.util.MimeUtil; import java.io.*; @@ -24,13 +27,50 @@ public class BinaryTempFileBody implements Body { } public void setEncoding(String encoding) throws MessagingException { - mEncoding = encoding; + if (mEncoding != null && mEncoding.equalsIgnoreCase(encoding)) { + return; + } + + // The encoding changed, so we need to convert the message + if (!MimeUtil.ENC_8BIT.equalsIgnoreCase(mEncoding)) { + throw new RuntimeException("Can't convert from encoding: " + mEncoding); + } + + try { + File newFile = File.createTempFile("body", null, mTempDirectory); + OutputStream out = new FileOutputStream(newFile); + try { + if (MimeUtil.ENC_QUOTED_PRINTABLE.equals(encoding)) { + out = new QuotedPrintableOutputStream(out, false); + } else if (MimeUtil.ENC_BASE64.equals(encoding)) { + out = new Base64OutputStream(out); + } else { + throw new RuntimeException("Target encoding not supported: " + encoding); + } + + InputStream in = getInputStream(); + try { + IOUtils.copy(in, out); + } finally { + in.close(); + } + } finally { + out.close(); + } + + mFile = newFile; + mEncoding = encoding; + } catch (IOException e) { + throw new MessagingException("Unable to convert body", e); + } } - public BinaryTempFileBody() { + public BinaryTempFileBody(String encoding) { if (mTempDirectory == null) { throw new RuntimeException("setTempDirectory has not been called on BinaryTempFileBody!"); } + + mEncoding = encoding; } public OutputStream getOutputStream() throws IOException { diff --git a/src/com/fsck/k9/mail/internet/BinaryTempFileMessageBody.java b/src/com/fsck/k9/mail/internet/BinaryTempFileMessageBody.java index c1503b342..e5fe7c370 100644 --- a/src/com/fsck/k9/mail/internet/BinaryTempFileMessageBody.java +++ b/src/com/fsck/k9/mail/internet/BinaryTempFileMessageBody.java @@ -18,6 +18,10 @@ import com.fsck.k9.mail.MessagingException; */ public class BinaryTempFileMessageBody extends BinaryTempFileBody implements CompositeBody { + public BinaryTempFileMessageBody(String encoding) { + super(encoding); + } + @Override public void setEncoding(String encoding) throws MessagingException { if (!MimeUtil.ENC_7BIT.equalsIgnoreCase(encoding) diff --git a/src/com/fsck/k9/mail/internet/MimeUtility.java b/src/com/fsck/k9/mail/internet/MimeUtility.java index cdd63789e..b78f8e378 100644 --- a/src/com/fsck/k9/mail/internet/MimeUtility.java +++ b/src/com/fsck/k9/mail/internet/MimeUtility.java @@ -1136,11 +1136,10 @@ public class MimeUtility { BinaryTempFileBody tempBody; if (MimeUtil.isMessage(contentType)) { - tempBody = new BinaryTempFileMessageBody(); + tempBody = new BinaryTempFileMessageBody(contentTransferEncoding); } else { - tempBody = new BinaryTempFileBody(); + tempBody = new BinaryTempFileBody(contentTransferEncoding); } - tempBody.setEncoding(contentTransferEncoding); OutputStream out = tempBody.getOutputStream(); try { diff --git a/tests/src/com/fsck/k9/mail/MessageTest.java b/tests/src/com/fsck/k9/mail/MessageTest.java index c5fc39c74..30a8852b4 100644 --- a/tests/src/com/fsck/k9/mail/MessageTest.java +++ b/tests/src/com/fsck/k9/mail/MessageTest.java @@ -291,8 +291,7 @@ public class MessageTest extends AndroidTestCase { private MimeMessage nestedMessage(MimeMessage subMessage) throws MessagingException, IOException { - BinaryTempFileMessageBody tempMessageBody = new BinaryTempFileMessageBody(); - tempMessageBody.setEncoding(MimeUtil.ENC_8BIT); + BinaryTempFileMessageBody tempMessageBody = new BinaryTempFileMessageBody(MimeUtil.ENC_8BIT); OutputStream out = tempMessageBody.getOutputStream(); try { @@ -334,7 +333,7 @@ public class MessageTest extends AndroidTestCase { String encodedTestString = "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "abcdefghijklmnopqrstuvwxyz0123456789+/\r\n"; - BinaryTempFileBody tempFileBody = new BinaryTempFileBody(); + BinaryTempFileBody tempFileBody = new BinaryTempFileBody(MimeUtil.ENC_BASE64); InputStream in = new ByteArrayInputStream( encodedTestString.getBytes("UTF-8")); From d1d7b60a093407e02259a2b88b9bfd4084005323 Mon Sep 17 00:00:00 2001 From: cketti Date: Wed, 26 Nov 2014 01:11:26 +0100 Subject: [PATCH 08/10] Add helper method to decode message bodies Depending on whether a Body implements RawDataBody (which indicates the class retains the original encoding) the helper method either strips the transfer encoding or simply returns the result of Body.getInputStream(). This should restore the original functionality. So saving messages in the database should work fine again. --- .../k9/mail/internet/BinaryTempFileBody.java | 7 ++- .../fsck/k9/mail/internet/MimeUtility.java | 49 ++++++++++++++++++- .../fsck/k9/mail/internet/RawDataBody.java | 12 +++++ .../fsck/k9/mail/store/local/LocalFolder.java | 2 +- .../mail/internet/MimeMessageParseTest.java | 8 +-- 5 files changed, 70 insertions(+), 8 deletions(-) create mode 100644 src/com/fsck/k9/mail/internet/RawDataBody.java diff --git a/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java b/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java index 677739bbf..17870c140 100644 --- a/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java +++ b/src/com/fsck/k9/mail/internet/BinaryTempFileBody.java @@ -15,7 +15,7 @@ import java.io.*; * and writeTo one time. After writeTo is called, or the InputStream returned from * getInputStream is closed the file is deleted and the Body should be considered disposed of. */ -public class BinaryTempFileBody implements Body { +public class BinaryTempFileBody implements RawDataBody { private static File mTempDirectory; private File mFile; @@ -26,6 +26,11 @@ public class BinaryTempFileBody implements Body { mTempDirectory = tempDirectory; } + @Override + public String getEncoding() { + return mEncoding; + } + public void setEncoding(String encoding) throws MessagingException { if (mEncoding != null && mEncoding.equalsIgnoreCase(encoding)) { return; diff --git a/src/com/fsck/k9/mail/internet/MimeUtility.java b/src/com/fsck/k9/mail/internet/MimeUtility.java index b78f8e378..da3dca5bd 100644 --- a/src/com/fsck/k9/mail/internet/MimeUtility.java +++ b/src/com/fsck/k9/mail/internet/MimeUtility.java @@ -2,6 +2,7 @@ package com.fsck.k9.mail.internet; import android.content.Context; +import android.util.Base64; import android.util.Log; import com.fsck.k9.K9; import com.fsck.k9.R; @@ -10,6 +11,7 @@ import com.fsck.k9.mail.*; import com.fsck.k9.mail.Message.RecipientType; import com.fsck.k9.mail.internet.BinaryTempFileBody.BinaryTempFileBodyInputStream; +import com.fsck.k9.view.MessageHeader; import org.apache.commons.io.IOUtils; import org.apache.james.mime4j.codec.Base64InputStream; import org.apache.james.mime4j.codec.QuotedPrintableInputStream; @@ -1026,7 +1028,7 @@ public class MimeUtility { * determine the charset from HTML message. */ if (mimeType.equalsIgnoreCase("text/html") && charset == null) { - InputStream in = part.getBody().getInputStream(); + InputStream in = MimeUtility.decodeBody(part.getBody()); try { byte[] buf = new byte[256]; in.read(buf, 0, buf.length); @@ -1062,7 +1064,7 @@ public class MimeUtility { * Now we read the part into a buffer for further processing. Because * the stream is now wrapped we'll remove any transfer encoding at this point. */ - InputStream in = part.getBody().getInputStream(); + InputStream in = MimeUtility.decodeBody(part.getBody()); try { String text = readToString(in, charset); @@ -1151,6 +1153,49 @@ public class MimeUtility { return tempBody; } + /** + * Get decoded contents of a body. + *

+ * Right now only some classes retain the original encoding of the body contents. Those classes have to implement + * the {@link RawDataBody} interface in order for this method to decode the data delivered by + * {@link Body#getInputStream()}. + *

+ * The ultimate goal is to get to a point where all classes retain the original data and {@code RawDataBody} can be + * merged into {@link Body}. + */ + public static InputStream decodeBody(Body body) throws MessagingException { + InputStream inputStream; + if (body instanceof RawDataBody) { + RawDataBody rawDataBody = (RawDataBody) body; + String encoding = rawDataBody.getEncoding(); + final InputStream rawInputStream = rawDataBody.getInputStream(); + if (MimeUtil.ENC_7BIT.equalsIgnoreCase(encoding) || MimeUtil.ENC_8BIT.equalsIgnoreCase(encoding)) { + inputStream = rawInputStream; + } else if (MimeUtil.ENC_BASE64.equalsIgnoreCase(encoding)) { + inputStream = new Base64InputStream(rawInputStream, false) { + @Override + public void close() throws IOException { + super.close(); + rawInputStream.close(); + } + }; + } else if (MimeUtil.ENC_QUOTED_PRINTABLE.equalsIgnoreCase(encoding)) { + inputStream = new QuotedPrintableInputStream(rawInputStream) { + @Override + public void close() throws IOException { + super.close(); + rawInputStream.close(); + } + }; + } else { + throw new RuntimeException("Encoding for RawDataBody not supported: " + encoding); + } + } else { + inputStream = body.getInputStream(); + } + + return inputStream; + } /** * Empty base class for the class hierarchy used by diff --git a/src/com/fsck/k9/mail/internet/RawDataBody.java b/src/com/fsck/k9/mail/internet/RawDataBody.java new file mode 100644 index 000000000..e3dee616e --- /dev/null +++ b/src/com/fsck/k9/mail/internet/RawDataBody.java @@ -0,0 +1,12 @@ +package com.fsck.k9.mail.internet; + + +import com.fsck.k9.mail.Body; + + +/** + * See {@link MimeUtility#decodeBody(Body)} + */ +public interface RawDataBody extends Body { + String getEncoding(); +} diff --git a/src/com/fsck/k9/mail/store/local/LocalFolder.java b/src/com/fsck/k9/mail/store/local/LocalFolder.java index e84098183..e264ef877 100644 --- a/src/com/fsck/k9/mail/store/local/LocalFolder.java +++ b/src/com/fsck/k9/mail/store/local/LocalFolder.java @@ -1561,7 +1561,7 @@ public class LocalFolder extends Folder implements Serializable { * If the attachment has a body we're expected to save it into the local store * so we copy the data into a cached attachment file. */ - InputStream in = attachment.getBody().getInputStream(); + InputStream in = MimeUtility.decodeBody(attachment.getBody()); try { tempAttachmentFile = File.createTempFile("att", null, attachmentDirectory); FileOutputStream out = new FileOutputStream(tempAttachmentFile); diff --git a/tests/src/com/fsck/k9/mail/internet/MimeMessageParseTest.java b/tests/src/com/fsck/k9/mail/internet/MimeMessageParseTest.java index 4558d4264..4107a7490 100644 --- a/tests/src/com/fsck/k9/mail/internet/MimeMessageParseTest.java +++ b/tests/src/com/fsck/k9/mail/internet/MimeMessageParseTest.java @@ -63,7 +63,7 @@ public class MimeMessageParseTest extends AndroidTestCase { private static void checkLeafParts(MimeMessage msg, String... expectedParts) throws Exception { List actual = new ArrayList(); for (Body leaf : getLeafParts(msg.getBody())) { - actual.add(streamToString(leaf.getInputStream())); + actual.add(streamToString(MimeUtility.decodeBody(leaf))); } assertEquals(Arrays.asList(expectedParts), actual); } @@ -83,7 +83,7 @@ public class MimeMessageParseTest extends AndroidTestCase { checkAddresses(msg.getRecipients(RecipientType.TO), "eva@example.org"); assertEquals("Testmail", msg.getSubject()); assertEquals("text/plain", msg.getContentType()); - assertEquals("this is some test text.", streamToString(msg.getBody().getInputStream())); + assertEquals("this is some test text.", streamToString(MimeUtility.decodeBody(msg.getBody()))); } public static void testSinglePart8BitRecurse() throws Exception { @@ -101,7 +101,7 @@ public class MimeMessageParseTest extends AndroidTestCase { checkAddresses(msg.getRecipients(RecipientType.TO), "eva@example.org"); assertEquals("Testmail", msg.getSubject()); assertEquals("text/plain; encoding=ISO-8859-1", msg.getContentType()); - assertEquals("gefährliche Umlaute", streamToString(msg.getBody().getInputStream())); + assertEquals("gefährliche Umlaute", streamToString(MimeUtility.decodeBody(msg.getBody()))); } public static void testSinglePartBase64NoRecurse() throws Exception { @@ -119,7 +119,7 @@ public class MimeMessageParseTest extends AndroidTestCase { checkAddresses(msg.getRecipients(RecipientType.TO), "eva@example.org"); assertEquals("Testmail", msg.getSubject()); assertEquals("text/plain", msg.getContentType()); - assertEquals("this is some more test text.", streamToString(msg.getBody().getInputStream())); + assertEquals("this is some more test text.", streamToString(MimeUtility.decodeBody(msg.getBody()))); } public static void testMultipartSingleLayerNoRecurse() throws Exception { From 3919c9d2d6483f432b14814bfd44875423a872ca Mon Sep 17 00:00:00 2001 From: cketti Date: Sun, 7 Dec 2014 03:31:58 +0100 Subject: [PATCH 09/10] Save multi part epilogue in MimeMultipart --- src/com/fsck/k9/mail/internet/MimeMessage.java | 6 ++++++ src/com/fsck/k9/mail/internet/MimeMultipart.java | 8 ++++++++ tests/src/com/fsck/k9/mail/ReconstructMessageTest.java | 3 ++- 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/src/com/fsck/k9/mail/internet/MimeMessage.java b/src/com/fsck/k9/mail/internet/MimeMessage.java index 8eb3dfe95..857d1ad76 100644 --- a/src/com/fsck/k9/mail/internet/MimeMessage.java +++ b/src/com/fsck/k9/mail/internet/MimeMessage.java @@ -2,6 +2,7 @@ package com.fsck.k9.mail.internet; import java.io.BufferedWriter; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -14,6 +15,7 @@ import java.util.Locale; import java.util.Set; import java.util.UUID; +import org.apache.commons.io.IOUtils; import org.apache.james.mime4j.MimeException; import org.apache.james.mime4j.dom.field.DateTimeField; import org.apache.james.mime4j.field.DefaultFieldParser; @@ -575,6 +577,10 @@ public class MimeMessage extends Message { @Override public void epilogue(InputStream is) throws IOException { + expect(MimeMultipart.class); + ByteArrayOutputStream epilogue = new ByteArrayOutputStream(); + IOUtils.copy(is, epilogue); + ((MimeMultipart) stack.peek()).setEpilogue(epilogue.toByteArray()); } @Override diff --git a/src/com/fsck/k9/mail/internet/MimeMultipart.java b/src/com/fsck/k9/mail/internet/MimeMultipart.java index 124ca5e4c..6775fdaea 100644 --- a/src/com/fsck/k9/mail/internet/MimeMultipart.java +++ b/src/com/fsck/k9/mail/internet/MimeMultipart.java @@ -11,6 +11,7 @@ import java.util.Random; public class MimeMultipart extends Multipart { private String mPreamble; + private byte[] mEpilogue; private String mContentType; @@ -53,6 +54,10 @@ public class MimeMultipart extends Multipart { this.mPreamble = preamble; } + public void setEpilogue(byte[] epilogue) { + mEpilogue = epilogue; + } + @Override public String getContentType() { return mContentType; @@ -90,6 +95,9 @@ public class MimeMultipart extends Multipart { writer.write(mBoundary); writer.write("--\r\n"); writer.flush(); + if (mEpilogue != null) { + out.write(mEpilogue); + } } @Override diff --git a/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java b/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java index 1e6528c99..e36854354 100644 --- a/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java +++ b/tests/src/com/fsck/k9/mail/ReconstructMessageTest.java @@ -42,7 +42,8 @@ public class ReconstructMessageTest extends AndroidTestCase { "VGhpcyBpcyBhIHRl\r\n" + "c3QgbWVzc2FnZQ==\r\n" + "\r\n" + - "------Boundary--\r\n"; + "------Boundary--\r\n" + + "Hi, I'm the epilogue"; BinaryTempFileBody.setTempDirectory(getContext().getCacheDir()); From e374538110fe3083f000b83f74451568941cd23a Mon Sep 17 00:00:00 2001 From: cketti Date: Mon, 8 Dec 2014 17:38:30 +0100 Subject: [PATCH 10/10] Store multi part preamble as byte array --- src/com/fsck/k9/mail/internet/MimeMessage.java | 9 +++------ src/com/fsck/k9/mail/internet/MimeMultipart.java | 10 +++------- 2 files changed, 6 insertions(+), 13 deletions(-) diff --git a/src/com/fsck/k9/mail/internet/MimeMessage.java b/src/com/fsck/k9/mail/internet/MimeMessage.java index 857d1ad76..a71e2a543 100644 --- a/src/com/fsck/k9/mail/internet/MimeMessage.java +++ b/src/com/fsck/k9/mail/internet/MimeMessage.java @@ -567,12 +567,9 @@ public class MimeMessage extends Message { @Override public void preamble(InputStream is) throws IOException { expect(MimeMultipart.class); - StringBuilder sb = new StringBuilder(); - int b; - while ((b = is.read()) != -1) { - sb.append((char)b); - } - ((MimeMultipart)stack.peek()).setPreamble(sb.toString()); + ByteArrayOutputStream preamble = new ByteArrayOutputStream(); + IOUtils.copy(is, preamble); + ((MimeMultipart)stack.peek()).setPreamble(preamble.toByteArray()); } @Override diff --git a/src/com/fsck/k9/mail/internet/MimeMultipart.java b/src/com/fsck/k9/mail/internet/MimeMultipart.java index 6775fdaea..d6ce4377a 100644 --- a/src/com/fsck/k9/mail/internet/MimeMultipart.java +++ b/src/com/fsck/k9/mail/internet/MimeMultipart.java @@ -10,7 +10,7 @@ import java.util.Locale; import java.util.Random; public class MimeMultipart extends Multipart { - private String mPreamble; + private byte[] mPreamble; private byte[] mEpilogue; private String mContentType; @@ -46,11 +46,7 @@ public class MimeMultipart extends Multipart { return sb.toString().toUpperCase(Locale.US); } - public String getPreamble() { - return mPreamble; - } - - public void setPreamble(String preamble) { + public void setPreamble(byte[] preamble) { this.mPreamble = preamble; } @@ -72,7 +68,7 @@ public class MimeMultipart extends Multipart { BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out), 1024); if (mPreamble != null) { - writer.write(mPreamble); + out.write(mPreamble); writer.write("\r\n"); }