Merge pull request #3169 from k9mail/rewrite_text_to_html_conversion

TextToHtml: Rewrite text to HTML conversion

k9mail/src/main/java/com/fsck/k9/message/html/DividerReplacer.kt

@@ -0,0 +1,27 @@
+package com.fsck.k9.message.html
+
+internal object DividerReplacer : TextToHtml.HtmlModifier {
+    private const val SIMPLE_DIVIDER = "[-=_]{3,}"
+    private const val ASCII_SCISSORS = "(?:-{2,}\\s?(?:>[%8]|[%8]<)\\s?-{2,})+"
+    private val PATTERN = Regex("(?:^|\\n)" +
+            "(?:" +
+                "\\s*" +
+                "(?:" + SIMPLE_DIVIDER + "|" + ASCII_SCISSORS + ")" +
+                "\\s*" +
+                "(?:\\n|$)" +
+            ")+")
+
+
+    override fun findModifications(text: CharSequence): List<HtmlModification> {
+        return PATTERN.findAll(text).map { matchResult ->
+            Divider(matchResult.range.start, matchResult.range.endInclusive + 1)
+        }.toList()
+    }
+
+
+    class Divider(startIndex: Int, endIndex: Int) : HtmlModification.Replace(startIndex, endIndex) {
+        override fun replace(textToHtml: TextToHtml) {
+            textToHtml.appendHtml("<hr>")
+        }
+    }
+}

k9mail/src/main/java/com/fsck/k9/message/html/EmailSectionExtractor.kt

@@ -96,12 +96,10 @@ class EmailSectionExtractor private constructor(val text: String) {
     }
 
     private fun completeLastSection() {
-        if (!isStartOfLine) {
+        if (quoteDepth == 0) {
-            if (quoteDepth == 0) {
+            sectionBuilder.addSegment(0, sectionStartIndex, text.length)
-                sectionBuilder.addSegment(0, sectionStartIndex, text.length)
+        } else if (!isStartOfLine) {
-            } else {
+            sectionBuilder.addSegment(spaces, startOfContentIndex, text.length)
-                sectionBuilder.addSegment(spaces, startOfContentIndex, text.length)
-            }
         }
 
         appendSection()

k9mail/src/main/java/com/fsck/k9/message/html/EmailTextToHtml.kt

@@ -0,0 +1,68 @@
+package com.fsck.k9.message.html
+
+class EmailTextToHtml private constructor(private val text: String) {
+    private val html = StringBuilder(text.length + EXTRA_BUFFER_LENGTH)
+    private var previousQuoteDepth = 0
+
+    fun convert(): String {
+        appendHtmlPrefix()
+
+        val sections = EmailSectionExtractor.extract(text)
+        sections.forEach { section ->
+            appendBlockQuoteElement(section.quoteDepth)
+
+            TextToHtml.appendAsHtmlFragment(html, section)
+        }
+
+        appendBlockQuoteElement(quoteDepth = 0)
+
+        appendHtmlSuffix()
+
+        return html.toString()
+    }
+
+    private fun appendHtmlPrefix() {
+        html.append("<pre class=\"$K9MAIL_CSS_CLASS\">")
+    }
+
+    private fun appendHtmlSuffix() {
+        html.append("</pre>")
+    }
+
+    private fun appendBlockQuoteElement(quoteDepth: Int) {
+        if (previousQuoteDepth > quoteDepth) {
+            repeat(previousQuoteDepth - quoteDepth) {
+                html.append("</blockquote>")
+            }
+        } else if (quoteDepth > previousQuoteDepth) {
+            for (depth in (previousQuoteDepth + 1)..quoteDepth) {
+                html.append("<blockquote " +
+                        "class=\"gmail_quote\" " +
+                        "style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid ")
+                html.append(quoteColor(depth))
+                html.append("; padding-left: 1ex;\">")
+            }
+        }
+        previousQuoteDepth = quoteDepth
+    }
+
+    private fun quoteColor(depth: Int): String = when (depth) {
+        1 -> "#729fcf"
+        2 -> "#ad7fa8"
+        3 -> "#8ae234"
+        4 -> "#fcaf3e"
+        5 -> "#e9b96e"
+        else -> "#ccc"
+    }
+
+
+    companion object {
+        private const val EXTRA_BUFFER_LENGTH = 2048
+        const val K9MAIL_CSS_CLASS = "k9mail"
+
+        @JvmStatic
+        fun convert(text: String): String {
+            return EmailTextToHtml(text).convert()
+        }
+    }
+}

k9mail/src/main/java/com/fsck/k9/message/html/HtmlConverter.java

@@ -5,7 +5,6 @@ import java.util.Collections;
 import java.util.HashSet;
 import java.util.Locale;
 import java.util.Set;
-import java.util.regex.Pattern;
 
 import android.text.Annotation;
 import android.text.Editable;
@@ -13,7 +12,6 @@ import android.text.Html;
 import android.text.Html.TagHandler;
 import android.text.Spannable;
 import android.text.Spanned;
-import android.text.TextUtils;
 
 import com.fsck.k9.K9;
 import org.xml.sax.XMLReader;
@@ -37,8 +35,6 @@ public class HtmlConverter {
     private static final char NBSP_CHARACTER = (char)0x00a0;    // utf-8 non-breaking space
     private static final char NBSP_REPLACEMENT = (char)0x20;    // space
 
-    // Number of extra bytes to allocate in a string buffer for htmlification.
-    private static final int TEXT_TO_HTML_EXTRA_BUFFER_LENGTH = 512;
 
     /**
      * Convert an HTML string to a plain text string.
@@ -128,254 +124,16 @@ public class HtmlConverter {
         }
     }
 
-    private static final int MAX_SMART_HTMLIFY_MESSAGE_LENGTH = 1024 * 256 ;
-
-    /**
-     * Naively convert a text string into an HTML document.
-     *
-     * <p>
-     * This method avoids using regular expressions on the entire message body to save memory.
-     * </p>
-     * <p>
-     * No HTML headers or footers are added to the result.  Headers and footers
-     * are added at display time in
-     * {@link com.fsck.k9.view#MessageWebView.setText(String) MessageWebView.setText()}
-     * </p>
-     *
-     * @param text
-     *         Plain text string.
-     * @return HTML string.
-     */
-    private static String simpleTextToHtml(String text) {
-        // Encode HTML entities to make sure we don't display something evil.
-        text = TextUtils.htmlEncode(text);
-
-        StringBuilder buff = new StringBuilder(text.length() + TEXT_TO_HTML_EXTRA_BUFFER_LENGTH);
-
-        buff.append(htmlifyMessageHeader());
-
-        for (int index = 0; index < text.length(); index++) {
-            char c = text.charAt(index);
-            switch (c) {
-            case '\n':
-                // pine treats <br> as two newlines, but <br/> as one newline.  Use <br/> so our messages aren't
-                // doublespaced.
-                buff.append("<br />");
-                break;
-            case '\r':
-                break;
-            default:
-                buff.append(c);
-            }//switch
-        }
-
-        buff.append(htmlifyMessageFooter());
-
-        return buff.toString();
-    }
-
-    private static final String HTML_BLOCKQUOTE_COLOR_TOKEN = "$$COLOR$$";
-    private static final String HTML_BLOCKQUOTE_START = "<blockquote class=\"gmail_quote\" " +
-            "style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid $$COLOR$$; padding-left: 1ex;\">";
-    private static final String HTML_BLOCKQUOTE_END = "</blockquote>";
-    private static final String HTML_NEWLINE = "<br />";
-    private static final Pattern ASCII_PATTERN_FOR_HR = Pattern.compile(
-            "(^|\\Q" + HTML_NEWLINE + "\\E)\\s*((\\Q" + HTML_NEWLINE + "\\E)*" +
-            "((((\\Q" + HTML_NEWLINE + "\\E){0,2}([-=_]{3,})(\\Q" + HTML_NEWLINE +
-            "\\E){0,2})|(([-=_]{2,} ?)(8&lt;|<gt>8|%&lt;|<gt>%)" +
-            "( ?[-=_]{2,})))+(\\Q" + HTML_NEWLINE + "\\E|$)))");
-
     /**
      * Convert a text string into an HTML document.
      *
      * <p>
-     * Attempts to do smart replacement for large documents to prevent OOM
-     * errors.
-     * <p>
      * No HTML headers or footers are added to the result.  Headers and footers
-     * are added at display time in
+     * are added at display time.
-     * {@link com.fsck.k9.view#MessageWebView.setText(String) MessageWebView.setText()}
      * </p>
-     * <p>
-     * To convert to a fragment, use {@link #textToHtmlFragment(String)} .
-     * </p>
-     *
-     * @param text
-     *         Plain text string.
-     * @return HTML string.
      */
     public static String textToHtml(String text) {
-        // Our HTMLification code is somewhat memory intensive
+        return EmailTextToHtml.convert(text);
-        // and was causing lots of OOM errors on the market
-        // if the message is big and plain text, just do
-        // a trivial htmlification
-        if (text.length() > MAX_SMART_HTMLIFY_MESSAGE_LENGTH) {
-            return simpleTextToHtml(text);
-        }
-        StringBuilder buff = new StringBuilder(text.length() + TEXT_TO_HTML_EXTRA_BUFFER_LENGTH);
-        boolean isStartOfLine = true;  // Are we currently at the start of a line?
-        int spaces = 0;
-        int quoteDepth = 0; // Number of DIVs deep we are.
-        int quotesThisLine = 0; // How deep we should be quoting for this line.
-        for (int index = 0; index < text.length(); index++) {
-            char c = text.charAt(index);
-            if (isStartOfLine) {
-                switch (c) {
-                case ' ':
-                    spaces++;
-                    break;
-                case '>':
-                    quotesThisLine++;
-                    spaces = 0;
-                    break;
-                case '\n':
-                    appendbq(buff, quotesThisLine, quoteDepth);
-                    quoteDepth = quotesThisLine;
-
-                    appendsp(buff, spaces);
-                    spaces = 0;
-
-                    appendchar(buff, c);
-                    isStartOfLine = true;
-                    quotesThisLine = 0;
-                    break;
-                default:
-                    isStartOfLine = false;
-
-                    appendbq(buff, quotesThisLine, quoteDepth);
-                    quoteDepth = quotesThisLine;
-
-                    appendsp(buff, spaces);
-                    spaces = 0;
-
-                    appendchar(buff, c);
-                    isStartOfLine = false;
-                    break;
-                }
-            }
-            else {
-                appendchar(buff, c);
-                if (c == '\n') {
-                    isStartOfLine = true;
-                    quotesThisLine = 0;
-                }
-            }
-        }
-        // Close off any quotes we may have opened.
-        if (quoteDepth > 0) {
-            for (int i = quoteDepth; i > 0; i--) {
-                buff.append(HTML_BLOCKQUOTE_END);
-            }
-        }
-        text = buff.toString();
-
-        // Make newlines at the end of blockquotes nicer by putting newlines beyond the first one outside of the
-        // blockquote.
-        text = text.replaceAll(
-                   "\\Q" + HTML_NEWLINE + "\\E((\\Q" + HTML_NEWLINE + "\\E)+?)\\Q" + HTML_BLOCKQUOTE_END + "\\E",
-                   HTML_BLOCKQUOTE_END + "$1"
-               );
-
-        text = ASCII_PATTERN_FOR_HR.matcher(text).replaceAll("<hr>");
-
-        StringBuffer sb = new StringBuffer(text.length() + TEXT_TO_HTML_EXTRA_BUFFER_LENGTH);
-
-        sb.append(htmlifyMessageHeader());
-        UriLinkifier.linkifyText(text, sb);
-        sb.append(htmlifyMessageFooter());
-
-        text = sb.toString();
-
-        // Above we replaced > with <gt>, now make it &gt;
-        text = text.replaceAll("<gt>", "&gt;");
-
-        return text;
-    }
-
-    private static void appendchar(StringBuilder buff, int c) {
-        switch (c) {
-        case '&':
-            buff.append("&amp;");
-            break;
-        case '<':
-            buff.append("&lt;");
-            break;
-        case '>':
-            // We use a token here which can't occur in htmlified text because &gt; is valid
-            // within links (where > is not), and linkifying links will include it if we
-            // do it here. We'll make another pass and change this back to &gt; after
-            // the linkification is done.
-            buff.append("<gt>");
-            break;
-        case '\r':
-            break;
-        case '\n':
-            // pine treats <br> as two newlines, but <br/> as one newline.  Use <br/> so our messages aren't
-            // doublespaced.
-            buff.append(HTML_NEWLINE);
-            break;
-        default:
-            buff.append((char)c);
-            break;
-        }
-    }
-
-    private static void appendsp(StringBuilder buff, int spaces) {
-        while (spaces > 0) {
-            buff.append(' ');
-            spaces--;
-        }
-    }
-
-    private static void appendbq(StringBuilder buff, int quotesThisLine, int quoteDepth) {
-        // Add/remove blockquotes by comparing this line's quotes to the previous line's quotes.
-        if (quotesThisLine > quoteDepth) {
-            for (int i = quoteDepth; i < quotesThisLine; i++) {
-                buff.append(HTML_BLOCKQUOTE_START.replace(HTML_BLOCKQUOTE_COLOR_TOKEN, getQuoteColor(i + 1)));
-            }
-        } else if (quotesThisLine < quoteDepth) {
-            for (int i = quoteDepth; i > quotesThisLine; i--) {
-                buff.append(HTML_BLOCKQUOTE_END);
-            }
-        }
-    }
-
-    protected static final String QUOTE_COLOR_DEFAULT = "#ccc";
-    protected static final String QUOTE_COLOR_LEVEL_1 = "#729fcf";
-    protected static final String QUOTE_COLOR_LEVEL_2 = "#ad7fa8";
-    protected static final String QUOTE_COLOR_LEVEL_3 = "#8ae234";
-    protected static final String QUOTE_COLOR_LEVEL_4 = "#fcaf3e";
-    protected static final String QUOTE_COLOR_LEVEL_5 = "#e9b96e";
-    private static final String K9MAIL_CSS_CLASS = "k9mail";
-
-    /**
-     * Return an HTML hex color string for a given quote level.
-     * @param level Quote level
-     * @return Hex color string with prepended #.
-     */
-    protected static String getQuoteColor(final int level) {
-        switch(level) {
-            case 1:
-                return QUOTE_COLOR_LEVEL_1;
-            case 2:
-                return QUOTE_COLOR_LEVEL_2;
-            case 3:
-                return QUOTE_COLOR_LEVEL_3;
-            case 4:
-                return QUOTE_COLOR_LEVEL_4;
-            case 5:
-                return QUOTE_COLOR_LEVEL_5;
-            default:
-                return QUOTE_COLOR_DEFAULT;
-        }
-    }
-
-    private static String htmlifyMessageHeader() {
-        return "<pre class=\"" + K9MAIL_CSS_CLASS + "\">";
-    }
-
-    private static String htmlifyMessageFooter() {
-        return "</pre>";
     }
 
     public static String wrapStatusMessage(CharSequence status) {
@@ -419,29 +177,16 @@ public class HtmlConverter {
         final String font = K9.messageViewFixedWidthFont()
                 ? "monospace"
                 : "sans-serif";
-        return "<style type=\"text/css\"> pre." + K9MAIL_CSS_CLASS +
+        return "<style type=\"text/css\"> pre." + EmailTextToHtml.K9MAIL_CSS_CLASS +
                 " {white-space: pre-wrap; word-wrap:break-word; " +
                 "font-family: " + font + "; margin-top: 0px}</style>";
     }
 
     /**
      * Convert a plain text string into an HTML fragment.
-     * @param text Plain text.
-     * @return HTML fragment.
      */
-    public static String textToHtmlFragment(final String text) {
+    public static String textToHtmlFragment(String text) {
-        // Escape the entities and add newlines.
+        return TextToHtml.toHtmlFragment(text);
-        String htmlified = TextUtils.htmlEncode(text);
-
-        // Linkify the message.
-        StringBuffer linkified = new StringBuffer(htmlified.length() + TEXT_TO_HTML_EXTRA_BUFFER_LENGTH);
-        UriLinkifier.linkifyText(htmlified, linkified);
-
-        // Add newlines and unescaping.
-        //
-        // For some reason, TextUtils.htmlEncode escapes ' into &apos;, which is technically part of the XHTML 1.0
-        // standard, but Gmail doesn't recognize it as an HTML entity. We unescape that here.
-        return linkified.toString().replaceAll("\r?\n", "<br>\r\n").replace("&apos;", "&#39;");
     }
 
     /**

k9mail/src/main/java/com/fsck/k9/message/html/HtmlModification.kt

@@ -0,0 +1,12 @@
+package com.fsck.k9.message.html
+
+internal abstract class HtmlModification private constructor(val startIndex: Int, val endIndex: Int) {
+    abstract class Wrap(startIndex: Int, endIndex: Int) : HtmlModification(startIndex, endIndex) {
+        abstract fun appendPrefix(textToHtml: TextToHtml)
+        abstract fun appendSuffix(textToHtml: TextToHtml)
+    }
+
+    abstract class Replace(startIndex: Int, endIndex: Int) : HtmlModification(startIndex, endIndex) {
+        abstract fun replace(textToHtml: TextToHtml)
+    }
+}

k9mail/src/main/java/com/fsck/k9/message/html/TextToHtml.kt

@@ -0,0 +1,83 @@
+package com.fsck.k9.message.html
+
+class TextToHtml private constructor(private val text: CharSequence, private val html: StringBuilder) {
+    fun appendAsHtmlFragment() {
+        val modifications = HTML_MODIFIERS
+                .flatMap { it.findModifications(text) }
+                .sortedBy { it.startIndex }
+
+        var currentIndex = 0
+        modifications.forEach { modification ->
+            appendHtmlEncoded(currentIndex, modification.startIndex)
+
+            when (modification) {
+                is HtmlModification.Wrap -> {
+                    modification.appendPrefix(this)
+                    appendHtmlEncoded(modification.startIndex, modification.endIndex)
+                    modification.appendSuffix(this)
+                }
+                is HtmlModification.Replace -> {
+                    modification.replace(this)
+                }
+            }
+
+            currentIndex = modification.endIndex
+        }
+
+        appendHtmlEncoded(currentIndex, text.length)
+    }
+
+    private fun appendHtmlEncoded(startIndex: Int, endIndex: Int) {
+        for (i in startIndex until endIndex) {
+            appendHtmlEncoded(text[i])
+        }
+    }
+
+    internal fun appendHtml(text: String) {
+        html.append(text)
+    }
+
+    internal fun appendHtmlEncoded(ch: Char) {
+        when (ch) {
+            '&' -> html.append("&")
+            '<' -> html.append("&lt;")
+            '>' -> html.append("&gt;")
+            '\r' -> Unit
+            '\n' -> html.append(TextToHtml.HTML_NEWLINE)
+            else -> html.append(ch)
+        }
+    }
+
+    internal fun appendHtmlAttributeEncoded(attributeValue: CharSequence) {
+        for (ch in attributeValue) {
+            when (ch) {
+                '&' -> html.append("&amp;")
+                '<' -> html.append("&lt;")
+                '"' -> html.append("&quot;")
+                else -> html.append(ch)
+            }
+        }
+    }
+
+    companion object {
+        private val HTML_MODIFIERS = listOf(DividerReplacer, UriLinkifier)
+        private const val HTML_NEWLINE = "<br>"
+        private const val TEXT_TO_HTML_EXTRA_BUFFER_LENGTH = 512
+
+        @JvmStatic
+        fun appendAsHtmlFragment(html: StringBuilder, text: CharSequence) {
+            TextToHtml(text, html).appendAsHtmlFragment()
+        }
+
+        @JvmStatic
+        fun toHtmlFragment(text: CharSequence): String {
+            val html = StringBuilder(text.length + TEXT_TO_HTML_EXTRA_BUFFER_LENGTH)
+            TextToHtml(text, html).appendAsHtmlFragment()
+            return html.toString()
+        }
+    }
+
+    internal interface HtmlModifier {
+        fun findModifications(text: CharSequence): List<HtmlModification>
+    }
+}

k9mail/src/main/java/com/fsck/k9/message/html/UriLinkifier.kt

@@ -1,31 +1,27 @@
 package com.fsck.k9.message.html
 
-
+internal object UriLinkifier : TextToHtml.HtmlModifier {
-@Deprecated("Helper to be able to transition to the new text to HTML conversion in smaller steps")
+    override fun findModifications(text: CharSequence): List<HtmlModification> {
-object UriLinkifier {
+        return UriMatcher.findUris(text).map {
-    @JvmStatic
+            LinkifyUri(it.startIndex, it.endIndex, it.uri)
-    fun linkifyText(text: String, html: StringBuffer) {
-        val uriMatches = UriMatcher.findUris(text)
-
-        var currentIndex = 0
-        uriMatches.forEach { uriMatch ->
-            append(html, text, currentIndex, uriMatch.startIndex)
-
-            html.append("<a href=\"")
-            html.append(uriMatch.uri)
-            html.append("\">")
-            html.append(uriMatch.uri)
-            html.append("</a>")
-
-            currentIndex = uriMatch.endIndex
         }
-
-        append(html, text, currentIndex, text.length)
     }
 
-    private fun append(html: StringBuffer, text: String, startIndex: Int, endIndex: Int) {
+
-        for (i in startIndex until endIndex) {
+    class LinkifyUri(
-            html.append(text[i])
+            startIndex: Int,
+            endIndex: Int,
+            val uri: CharSequence
+    ) : HtmlModification.Wrap(startIndex, endIndex) {
+
+        override fun appendPrefix(textToHtml: TextToHtml) {
+            textToHtml.appendHtml("<a href=\"")
+            textToHtml.appendHtmlAttributeEncoded(uri)
+            textToHtml.appendHtml("\">")
+        }
+
+        override fun appendSuffix(textToHtml: TextToHtml) {
+            textToHtml.appendHtml("</a>")
         }
     }
 }

k9mail/src/main/java/com/fsck/k9/message/html/UriMatcher.kt

@@ -13,8 +13,7 @@ object UriMatcher {
         )
     }.invoke(HttpUriParser())
 
-    // FIXME: Remove > once the text to HTML code has been replaced
+    private const val SCHEME_SEPARATORS = " (\\n<"
-    private const val SCHEME_SEPARATORS = " (\\n<>"
     private const val ALLOWED_SEPARATORS_PATTERN = "(?:^|[$SCHEME_SEPARATORS])"
     private val URI_SCHEME = Regex(
             "$ALLOWED_SEPARATORS_PATTERN(${ SUPPORTED_URIS.keys.joinToString("|") })",

k9mail/src/test/java/com/fsck/k9/mailstore/MessageViewInfoExtractorTest.java

@@ -151,7 +151,7 @@ public class MessageViewInfoExtractorTest {
                 "not flowed line";
         String expectedHtml =
                 "<pre class=\"k9mail\">" +
-                        "K-9 Mail rocks :&gt; flowed line<br />not flowed line" +
+                        "K-9 Mail rocks :&gt; flowed line<br>not flowed line" +
                         "</pre>";
 
         assertEquals(expectedText, container.text);
@@ -353,12 +353,12 @@ public class MessageViewInfoExtractorTest {
         String expectedHtmlText = "<table style=\"border: 0\">" +
                 "<tr><th style=\"text-align: left; vertical-align: top;\">Subject:</th><td>(No subject)</td></tr>" +
                 "</table>" +
-                "<pre class=\"k9mail\">text body of first message<br /></pre>" +
+                "<pre class=\"k9mail\">text body of first message<br></pre>" +
                 "<p style=\"margin-top: 2.5em; margin-bottom: 1em; border-bottom: 1px solid #000\"></p>" +
                 "<table style=\"border: 0\">" +
                 "<tr><th style=\"text-align: left; vertical-align: top;\">Subject:</th><td>subject of second message</td></tr>" +
                 "</table>" +
-                "<pre class=\"k9mail\">text part of second message<br /></pre>";
+                "<pre class=\"k9mail\">text part of second message<br></pre>";
 
 
         assertEquals(4, outputViewableParts.size());

k9mail/src/test/java/com/fsck/k9/message/html/EmailSectionExtractorTest.kt

@@ -26,6 +26,19 @@ class EmailSectionExtractorTest {
         }
     }
 
+    @Test
+    fun simpleMessageEndingWithTwoNewlines() {
+        val message = "Hello\n\n"
+
+        val sections = EmailSectionExtractor.extract(message)
+
+        assertThat(sections.size).isEqualTo(1)
+        with(sections[0]) {
+            assertThat(quoteDepth).isEqualTo(0)
+            assertThat(toString()).isEqualTo(message)
+        }
+    }
+
     @Test
     fun quoteFollowedByReply() {
         val message = """
@@ -81,6 +94,24 @@ class EmailSectionExtractorTest {
         }
     }
 
+    @Test
+    fun quoteEndingWithEmptyLineButNoNewline() {
+        val message = """
+            > Quoted text
+            > """.trimIndent()
+
+        val sections = EmailSectionExtractor.extract(message)
+
+        assertThat(sections.size).isEqualTo(1)
+        with(sections[0]) {
+            assertThat(quoteDepth).isEqualTo(1)
+            // Note: "Quoted text\n\n" would be a better representation of the quoted text. The goal of this test is
+            // not to preserve the current behavior of only ending in one newline, but to make sure we don't add the
+            // last line twice.
+            assertThat(toString()).isEqualTo("Quoted text\n")
+        }
+    }
+
     @Test
     fun chaosQuoting() {
         val message = """

k9mail/src/test/java/com/fsck/k9/message/html/HtmlConverterTest.java

@@ -55,27 +55,27 @@ public class HtmlConverterTest {
         String result = HtmlConverter.textToHtml(message);
         writeToFile(result);
         assertEquals("<pre class=\"k9mail\">"
-                + "Panama!<br />"
+                + "Panama!<br>"
-                + "<br />"
+                + "<br>"
-                + "Bob Barker &lt;bob@aol.com&gt; wrote:<br />"
+                + "Bob Barker &lt;bob@aol.com&gt; wrote:<br>"
                 +
                 "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #729fcf; padding-left: 1ex;\">"
-                + " a canal<br />"
+                + " a canal<br>"
-                + "<br />"
+                + "<br>"
-                + " Dorothy Jo Gideon &lt;dorothy@aol.com&gt; espoused:<br />"
+                + " Dorothy Jo Gideon &lt;dorothy@aol.com&gt; espoused:<br>"
                 +
                 "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #ad7fa8; padding-left: 1ex;\">"
-                + "A man, a plan...<br />"
+                + "A man, a plan...<br>"
                 + "</blockquote>"
-                + " Too easy!<br />"
+                + "Too easy!<br>"
                 + "</blockquote>"
-                + "<br />"
+                + "<br>"
-                + "Nice job :)<br />"
+                + "Nice job :)<br>"
                 +
                 "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #729fcf; padding-left: 1ex;\">"
                 +
                 "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #ad7fa8; padding-left: 1ex;\">"
-                + " Guess!"
+                + "Guess!"
                 + "</blockquote>"
                 + "</blockquote>"
                 + "</pre>", result);
@@ -94,14 +94,14 @@ public class HtmlConverterTest {
         String result = HtmlConverter.textToHtml(message);
         writeToFile(result);
         assertEquals("<pre class=\"k9mail\">"
-                + "*facepalm*<br />"
+                + "*facepalm*<br>"
-                + "<br />"
+                + "<br>"
-                + "Bob Barker &lt;bob@aol.com&gt; wrote:<br />"
+                + "Bob Barker &lt;bob@aol.com&gt; wrote:<br>"
                 + "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #729fcf; padding-left: 1ex;\">"
-                +   " A wise man once said...<br />"
+                +   " A wise man once said...<br>"
-                +   "<br />"
+                +   "<br>"
-                +   "     LOL F1RST!!!!!<br />"
+                +   "     LOL F1RST!!!!!<br>"
-                +   "<br />"
+                +   "<br>"
                 +   " :)"
                 + "</blockquote></pre>", result);
 
@@ -109,16 +109,6 @@ public class HtmlConverterTest {
 
     @Test
     public void testQuoteDepthColor() {
-        assertEquals(HtmlConverter.getQuoteColor(1), HtmlConverter.QUOTE_COLOR_LEVEL_1);
-        assertEquals(HtmlConverter.getQuoteColor(2), HtmlConverter.QUOTE_COLOR_LEVEL_2);
-        assertEquals(HtmlConverter.getQuoteColor(3), HtmlConverter.QUOTE_COLOR_LEVEL_3);
-        assertEquals(HtmlConverter.getQuoteColor(4), HtmlConverter.QUOTE_COLOR_LEVEL_4);
-        assertEquals(HtmlConverter.getQuoteColor(5), HtmlConverter.QUOTE_COLOR_LEVEL_5);
-
-        assertEquals(HtmlConverter.getQuoteColor(-1), HtmlConverter.QUOTE_COLOR_DEFAULT);
-        assertEquals(HtmlConverter.getQuoteColor(0), HtmlConverter.QUOTE_COLOR_DEFAULT);
-        assertEquals(HtmlConverter.getQuoteColor(6), HtmlConverter.QUOTE_COLOR_DEFAULT);
-
         String message = "zero\r\n" +
                 "> one\r\n" +
                 ">> two\r\n" +
@@ -129,19 +119,19 @@ public class HtmlConverterTest {
         String result = HtmlConverter.textToHtml(message);
         writeToFile(result);
         assertEquals("<pre class=\"k9mail\">"
-                + "zero<br />"
+                + "zero<br>"
                 + "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #729fcf; padding-left: 1ex;\">"
-                +   " one<br />"
+                +   "one<br>"
                 +   "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #ad7fa8; padding-left: 1ex;\">"
-                +     " two<br />"
+                +     "two<br>"
                 +     "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #8ae234; padding-left: 1ex;\">"
-                +       " three<br />"
+                +       "three<br>"
                 +       "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #fcaf3e; padding-left: 1ex;\">"
-                +         " four<br />"
+                +         "four<br>"
                 +         "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #e9b96e; padding-left: 1ex;\">"
-                +           " five<br />"
+                +           "five<br>"
                 +           "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #ccc; padding-left: 1ex;\">"
-                +             " six"
+                +             "six"
                 +           "</blockquote>"
                 +         "</blockquote>"
                 +       "</blockquote>"
@@ -183,9 +173,9 @@ public class HtmlConverterTest {
         String result = HtmlConverter.textToHtml(message);
         writeToFile(result);
         assertEquals("<pre class=\"k9mail\">"
-                + "foo<br />"
+                + "foo<br>"
-                + " bar<br />"
+                + " bar<br>"
-                + "  baz<br />"
+                + "  baz<br>"
                 + "</pre>", result);
     }
 
@@ -200,12 +190,12 @@ public class HtmlConverterTest {
         String result = HtmlConverter.textToHtml(message);
         writeToFile(result);
         assertEquals("<pre class=\"k9mail\">"
-                + " <br />"
+                + " <br>"
-                + "  &amp;<br />"
+                + "  &amp;<br>"
-                + "    <br />"
+                + "    <br>"
-                + "   &lt;<br />"
+                + "   &lt;<br>"
                 + "<blockquote class=\"gmail_quote\" style=\"margin: 0pt 0pt 1ex 0.8ex; border-left: 1px solid #729fcf; padding-left: 1ex;\">"
-                + " <br />"
+                + "<br>"
                 + "</blockquote>"
                 + "</pre>", result);
     }
@@ -237,7 +227,7 @@ public class HtmlConverterTest {
     public void dashesContainingSpacesIgnoredAsHR() {
         String text = "hello\n--- --- --- --- ---\nfoo bar";
         String result = HtmlConverter.textToHtml(text);
-        assertEquals("<pre class=\"k9mail\">hello<br />--- --- --- --- ---<br />foo bar</pre>",
+        assertEquals("<pre class=\"k9mail\">hello<br>--- --- --- --- ---<br>foo bar</pre>",
                 result);
     }
 
@@ -252,28 +242,28 @@ public class HtmlConverterTest {
     public void dashedHorizontalRulePrefixedWithTextIgnoredAsHR() {
         String text = "hello----\n\n";
         String result = HtmlConverter.textToHtml(text);
-        assertEquals("<pre class=\"k9mail\">hello----<br /><br /></pre>", result);
+        assertEquals("<pre class=\"k9mail\">hello----<br><br></pre>", result);
     }
 
     @Test
     public void doubleMinusIgnoredAsHR() {
         String text = "--\n";
         String result = HtmlConverter.textToHtml(text);
-        assertEquals("<pre class=\"k9mail\">--<br /></pre>", result);
+        assertEquals("<pre class=\"k9mail\">--<br></pre>", result);
     }
 
     @Test
     public void doubleEqualsIgnoredAsHR() {
         String text = "==\n";
         String result = HtmlConverter.textToHtml(text);
-        assertEquals("<pre class=\"k9mail\">==<br /></pre>", result);
+        assertEquals("<pre class=\"k9mail\">==<br></pre>", result);
     }
 
     @Test
     public void doubleUnderscoreIgnoredAsHR() {
         String text = "__\n";
         String result = HtmlConverter.textToHtml(text);
-        assertEquals("<pre class=\"k9mail\">__<br /></pre>", result);
+        assertEquals("<pre class=\"k9mail\">__<br></pre>", result);
     }
 
     @Test
@@ -308,7 +298,7 @@ public class HtmlConverterTest {
     public void replacementOfScissorsByHR() {
         String text = "hello\n-- %< -------------- >8 --\nworld\n";
         String result = HtmlConverter.textToHtml(text);
-        assertEquals("<pre class=\"k9mail\">hello<hr>world<br /></pre>", result);
+        assertEquals("<pre class=\"k9mail\">hello<hr>world<br></pre>", result);
     }
 
     @Test

k9mail/src/test/java/com/fsck/k9/message/html/UriLinkifierTest.java

@@ -1,148 +0,0 @@
-package com.fsck.k9.message.html;
-
-
-import com.fsck.k9.K9RobolectricTestRunner;
-import org.junit.Test;
-import org.junit.runner.RunWith;
-import org.robolectric.annotation.Config;
-
-import static com.fsck.k9.message.html.UriParserTestHelper.assertLinkOnly;
-import static junit.framework.Assert.assertEquals;
-
-
-@RunWith(K9RobolectricTestRunner.class)
-@Config(manifest = Config.NONE)
-public class UriLinkifierTest {
-    private StringBuffer outputBuffer = new StringBuffer();
-
-
-    @Test
-    public void emptyText() {
-        String text = "";
-
-        UriLinkifier.linkifyText(text, outputBuffer);
-
-        assertEquals(text, outputBuffer.toString());
-    }
-
-    @Test
-    public void textWithoutUri_shouldBeCopiedToOutputBuffer() {
-        String text = "some text here";
-
-        UriLinkifier.linkifyText(text, outputBuffer);
-
-        assertEquals(text, outputBuffer.toString());
-    }
-
-    @Test
-    public void simpleUri() {
-        String uri = "http://example.org";
-
-        UriLinkifier.linkifyText(uri, outputBuffer);
-
-        assertLinkOnly(uri, outputBuffer);
-    }
-
-    @Test
-    public void uriPrecededBySpace() {
-        String text = " http://example.org";
-
-        UriLinkifier.linkifyText(text, outputBuffer);
-
-        assertEquals(" <a href=\"http://example.org\">http://example.org</a>", outputBuffer.toString());
-    }
-
-    @Test
-    public void uriPrecededByOpeningParenthesis() {
-        String text = "(http://example.org";
-
-        UriLinkifier.linkifyText(text, outputBuffer);
-
-        assertEquals("(<a href=\"http://example.org\">http://example.org</a>", outputBuffer.toString());
-    }
-
-    @Test
-    public void uriPrecededBySomeText() {
-        String uri = "Check out my fantastic URI: http://example.org";
-
-        UriLinkifier.linkifyText(uri, outputBuffer);
-
-        assertEquals("Check out my fantastic URI: <a href=\"http://example.org\">http://example.org</a>",
-                outputBuffer.toString());
-    }
-
-    @Test
-    public void uriWithTrailingText() {
-        String uri = "http://example.org/ is the best";
-
-        UriLinkifier.linkifyText(uri, outputBuffer);
-
-        assertEquals("<a href=\"http://example.org/\">http://example.org/</a> is the best", outputBuffer.toString());
-    }
-
-    @Test
-    public void uriEmbeddedInText() {
-        String uri = "prefix http://example.org/ suffix";
-
-        UriLinkifier.linkifyText(uri, outputBuffer);
-
-        assertEquals("prefix <a href=\"http://example.org/\">http://example.org/</a> suffix", outputBuffer.toString());
-    }
-
-    @Test
-    public void uriWithUppercaseScheme() {
-        String uri = "HTTP://example.org/";
-
-        UriLinkifier.linkifyText(uri, outputBuffer);
-
-        assertEquals("<a href=\"HTTP://example.org/\">HTTP://example.org/</a>", outputBuffer.toString());
-    }
-
-    @Test
-    public void uriNotPrecededByValidSeparator_shouldNotBeLinkified() {
-        String text = "myhttp://example.org";
-
-        UriLinkifier.linkifyText(text, outputBuffer);
-
-        assertEquals(text, outputBuffer.toString());
-    }
-
-    @Test
-    public void uriNotPrecededByValidSeparatorFollowedByValidUri() {
-        String text = "myhttp: http://example.org";
-
-        UriLinkifier.linkifyText(text, outputBuffer);
-
-        assertEquals("myhttp: <a href=\"http://example.org\">http://example.org</a>", outputBuffer.toString());
-    }
-
-    @Test
-    public void schemaMatchWithInvalidUriInMiddleOfTextFollowedByValidUri() {
-        String text = "prefix http:42 http://example.org";
-
-        UriLinkifier.linkifyText(text, outputBuffer);
-
-        assertEquals("prefix http:42 <a href=\"http://example.org\">http://example.org</a>", outputBuffer.toString());
-    }
-
-    @Test
-    public void multipleValidUrisInRow() {
-        String text = "prefix http://uri1.example.org some text http://uri2.example.org/path postfix";
-
-        UriLinkifier.linkifyText(text, outputBuffer);
-
-        assertEquals(
-                "prefix <a href=\"http://uri1.example.org\">http://uri1.example.org</a> some text " +
-                        "<a href=\"http://uri2.example.org/path\">http://uri2.example.org/path</a> postfix",
-                outputBuffer.toString());
-    }
-
-    @Test
-    public void uriSurroundedByHtmlTags() {
-        String text = "<br>http://uri.example.org<hr>";
-
-        UriLinkifier.linkifyText(text, outputBuffer);
-
-        assertEquals("<br><a href=\"http://uri.example.org\">http://uri.example.org</a><hr>", outputBuffer.toString());
-    }
-}

k9mail/src/test/java/com/fsck/k9/message/html/UriParserTestHelper.java

@@ -1,38 +0,0 @@
-package com.fsck.k9.message.html;
-
-
-import org.jsoup.Jsoup;
-import org.jsoup.nodes.Document;
-import org.jsoup.nodes.Element;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertTrue;
-
-
-public class UriParserTestHelper {
-    public static void assertContainsLink(String expected, StringBuffer actual) {
-        String linkifiedUri = actual.toString();
-        Document document = Jsoup.parseBodyFragment(linkifiedUri);
-        Element anchorElement = document.select("a").first();
-        assertNotNull("No <a> element found", anchorElement);
-        assertEquals(expected, anchorElement.text());
-        assertEquals(expected, anchorElement.attr("href"));
-    }
-
-    public static void assertLinkOnly(String expected, StringBuffer actual) {
-        String linkifiedUri = actual.toString();
-        Document document = Jsoup.parseBodyFragment(linkifiedUri);
-        Element anchorElement = document.select("a").first();
-        assertNotNull("No <a> element found", anchorElement);
-        assertEquals(expected, anchorElement.text());
-        assertEquals(expected, anchorElement.attr("href"));
-
-        assertAnchorElementIsSoleContent(document, anchorElement);
-    }
-
-    private static void assertAnchorElementIsSoleContent(Document document, Element anchorElement) {
-        assertEquals(document.body(), anchorElement.parent());
-        assertTrue("<a> element is surrounded by text", document.body().textNodes().isEmpty());
-    }
-}