Merge pull request #6301 from thundernest/utf8_in_imap_response

Add support for UTF-8 data in BODYSTRUCTURE response

mail/common/src/main/java/com/fsck/k9/mail/internet/MimeExtensions.kt

@@ -26,6 +26,7 @@ internal const val SEMICOLON = ';'
 internal const val EQUALS_SIGN = '='
 internal const val ASTERISK = '*'
 internal const val SINGLE_QUOTE = '\''
+internal const val BACKSLASH = '\\'
 
 internal fun Char.isTSpecial() = this in TSPECIALS
 

mail/common/src/main/java/com/fsck/k9/mail/internet/MimeParameterEncoder.kt

@@ -145,7 +145,7 @@ object MimeParameterEncoder {
 
     private fun String.isQuotable() = all { it.isQuotable() }
 
-    fun String.quoted(): String {
+    private fun String.quoted(): String {
         // quoted-string = [CFWS] DQUOTE *([FWS] qcontent) [FWS] DQUOTE [CFWS]
         // qcontent      = qtext / quoted-pair
         // quoted-pair   = ("\" (VCHAR / WSP))
@@ -165,6 +165,22 @@ object MimeParameterEncoder {
         }
     }
 
+    // RFC 6532-style header values
+    // Right now we only create such values for internal use (see IMAP BODYSTRUCTURE response parsing code)
+    fun String.quotedUtf8(): String {
+        return buildString(capacity = length + 16) {
+            append(DQUOTE)
+            for (c in this@quotedUtf8) {
+                if (c == DQUOTE || c == BACKSLASH) {
+                    append('\\').append(c)
+                } else {
+                    append(c)
+                }
+            }
+            append(DQUOTE)
+        }
+    }
+
     private fun String.quotedLength(): Int {
         var length = 2 /* start and end quote */
         for (c in this) {

mail/common/src/test/java/com/fsck/k9/mail/internet/MimeParameterDecoderTest.kt

@@ -378,6 +378,14 @@ class MimeParameterDecoderTest {
         assertThat(mimeValue.ignoredParameters).isEmpty()
     }
 
+    @Test
+    fun `UTF-8 data in header value`() {
+        val mimeValue = MimeParameterDecoder.decode("application/x-stuff; name=\"filenäme.ext\"")
+
+        assertThat(mimeValue.parameters).containsExactlyEntries("name" to "filenäme.ext")
+        assertThat(mimeValue.ignoredParameters).isEmpty()
+    }
+
     private fun MapSubject.containsExactlyEntries(vararg values: Pair<String, String>): Ordered {
         return containsExactlyEntriesIn(values.toMap())
     }

mail/protocols/imap/build.gradle

@@ -17,6 +17,7 @@ dependencies {
     implementation "com.jcraft:jzlib:1.0.7"
     implementation "com.beetstra.jutf7:jutf7:1.0.0"
     implementation "commons-io:commons-io:${versions.commonsIo}"
+    implementation "com.squareup.okio:okio:${versions.okio}"
 
     testImplementation project(":mail:testing")
     testImplementation "junit:junit:${versions.junit}"

mail/protocols/imap/src/main/java/com/fsck/k9/mail/store/imap/ImapResponseParser.java

@@ -10,6 +10,7 @@ import com.fsck.k9.logging.Timber;
 import com.fsck.k9.mail.K9MailLib;
 import com.fsck.k9.mail.filter.FixedLengthInputStream;
 import com.fsck.k9.mail.filter.PeekableInputStream;
+import okio.Buffer;
 
 import static com.fsck.k9.mail.K9MailLib.DEBUG_PROTOCOL_IMAP;
 
@@ -396,7 +397,7 @@ class ImapResponseParser {
     private String parseQuoted() throws IOException {
         expect('"');
 
-        StringBuilder sb = new StringBuilder();
+        Buffer buffer = new Buffer();
         int ch;
         boolean escape = false;
         while ((ch = inputStream.read()) != -1) {
@@ -404,12 +405,13 @@ class ImapResponseParser {
                 // Found the escape character
                 escape = true;
             } else if (!escape && ch == '"') {
-                return sb.toString();
+                return buffer.readUtf8();
             } else {
-                sb.append((char) ch);
+                buffer.writeByte(ch);
                 escape = false;
             }
         }
+
         throw new IOException("parseQuoted(): end of stream reached");
     }
 

mail/protocols/imap/src/main/java/com/fsck/k9/mail/store/imap/RealImapFolder.kt

@@ -16,14 +16,12 @@ import com.fsck.k9.mail.internet.MimeHeader
 import com.fsck.k9.mail.internet.MimeMessageHelper
 import com.fsck.k9.mail.internet.MimeMultipart
 import com.fsck.k9.mail.internet.MimeParameterEncoder.isToken
-import com.fsck.k9.mail.internet.MimeParameterEncoder.quoted
+import com.fsck.k9.mail.internet.MimeParameterEncoder.quotedUtf8
 import com.fsck.k9.mail.internet.MimeUtility
 import java.io.IOException
 import java.io.InputStream
 import java.text.SimpleDateFormat
 import java.util.Date
-import java.util.HashMap
-import java.util.LinkedHashSet
 import java.util.Locale
 import kotlin.math.max
 import kotlin.math.min
@@ -891,7 +889,7 @@ internal class RealImapFolder(
                 for (i in bodyParams.indices step 2) {
                     val paramName = bodyParams.getString(i)
                     val paramValue = bodyParams.getString(i + 1)
-                    val encodedValue = if (paramValue.isToken()) paramValue else paramValue.quoted()
+                    val encodedValue = if (paramValue.isToken()) paramValue else paramValue.quotedUtf8()
                     contentType.append(String.format(";\r\n %s=%s", paramName, encodedValue))
                 }
             }
@@ -918,7 +916,7 @@ internal class RealImapFolder(
                     for (i in bodyDispositionParams.indices step 2) {
                         val paramName = bodyDispositionParams.getString(i).lowercase()
                         val paramValue = bodyDispositionParams.getString(i + 1)
-                        val encodedValue = if (paramValue.isToken()) paramValue else paramValue.quoted()
+                        val encodedValue = if (paramValue.isToken()) paramValue else paramValue.quotedUtf8()
                         contentDisposition.append(String.format(";\r\n %s=%s", paramName, encodedValue))
                     }
                 }

mail/protocols/imap/src/test/java/com/fsck/k9/mail/store/imap/ImapResponseParserTest.java

@@ -8,6 +8,7 @@ import java.util.List;
 
 import com.fsck.k9.mail.filter.FixedLengthInputStream;
 import com.fsck.k9.mail.filter.PeekableInputStream;
+import kotlin.text.Charsets;
 import org.junit.Test;
 
 import static java.util.Arrays.asList;
@@ -346,6 +347,16 @@ public class ImapResponseParserTest {
         assertEquals("qu\"oted", response.getString(0));
     }
 
+    @Test
+    public void utf8InQuotedString() throws Exception {
+        ImapResponseParser parser = createParser("* \"quöted\"\r\n");
+
+        ImapResponse response = parser.readResponse();
+
+        assertEquals(1, response.size());
+        assertEquals("quöted", response.getString(0));
+    }
+
     @Test(expected = IOException.class)
     public void testParseQuotedToEndOfStream() throws Exception {
         ImapResponseParser parser = createParser("* \"abc");
@@ -484,7 +495,7 @@ public class ImapResponseParserTest {
     }
 
     private ImapResponseParser createParser(String response) {
-        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(response.getBytes());
+        ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(response.getBytes(Charsets.UTF_8));
         peekableInputStream = new PeekableInputStream(byteArrayInputStream);
         return new ImapResponseParser(peekableInputStream);
     }

mail/protocols/imap/src/test/java/com/fsck/k9/mail/store/imap/RealImapFolderTest.kt

@@ -770,6 +770,15 @@ class RealImapFolderTest {
         )
     }
 
+    @Test
+    fun `fetch() with UTF-8 encoded content type parameter`() {
+        testHeaderFromBodyStructure(
+            bodyStructure = """("text" "plain" ("name" "filenäme.ext") NIL NIL "7bit" 42 23)""",
+            headerName = MimeHeader.HEADER_CONTENT_TYPE,
+            expectedHeaderValue = "text/plain;\r\n name=\"filenäme.ext\""
+        )
+    }
+
     @Test
     fun `fetch() with simple content disposition parameter`() {
         testHeaderFromBodyStructure(
@@ -810,6 +819,16 @@ class RealImapFolderTest {
         )
     }
 
+    @Test
+    fun `fetch() with UTF-8 encoded content disposition parameter`() {
+        testHeaderFromBodyStructure(
+            bodyStructure = """("application" "octet-stream" NIL NIL NIL "8bit" 23 NIL """ +
+                """("attachment" ("filename" "filenäme.ext")) NIL NIL)""",
+            headerName = MimeHeader.HEADER_CONTENT_DISPOSITION,
+            expectedHeaderValue = "attachment;\r\n filename=\"filenäme.ext\";\r\n size=23"
+        )
+    }
+
     @Test
     fun fetch_withBodySaneFetchProfile_shouldIssueRespectiveCommand() {
         val folder = createFolder("Folder")