Add one pass URI parser/linkifier

This commit is contained in:
Tim Bolender 2017-03-13 19:39:59 +01:00 committed by cketti
parent 93589171d8
commit 0d3d9aab32
8 changed files with 771 additions and 111 deletions

View file

@ -0,0 +1,33 @@
package com.fsck.k9.message.html;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Parses and "linkifies" bitcoin links.
*/
class BitcoinUriParser implements UriParser {
private static final Pattern BITCOIN_URI_PATTERN =
Pattern.compile("bitcoin:[1-9a-km-zA-HJ-NP-Z]{27,34}(\\?[a-zA-Z0-9$\\-_.+!*'(),%:@&=]*)?");
@Override
public int linkifyUri(String text, int startPos, StringBuffer outputBuffer) {
Matcher matcher = BITCOIN_URI_PATTERN.matcher(text);
// Skip not matching uris
if (!matcher.find(startPos) || matcher.start() != startPos) {
return startPos;
}
String bitcoinUri = matcher.group();
outputBuffer.append("<a href=\"")
.append(bitcoinUri)
.append("\">")
.append(bitcoinUri)
.append("</a>");
return matcher.end();
}
}

View file

@ -1,85 +1,30 @@
package com.fsck.k9.message.html;
import android.text.*;
import android.text.Html.TagHandler;
import com.fsck.k9.K9;
import org.xml.sax.XMLReader;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import android.text.Annotation;
import android.text.Editable;
import android.text.Html;
import android.text.Html.TagHandler;
import android.text.Spannable;
import android.text.Spanned;
import android.text.TextUtils;
import com.fsck.k9.K9;
import org.xml.sax.XMLReader;
/**
* Contains common routines to convert html to text and vice versa.
*/
public class HtmlConverter {
/* This comprises most common used Unicode characters allowed in IRI
* as detailed in RFC 3987.
* Specifically, those two byte Unicode characters are not included.
*/
private static final String GOOD_IRI_CHAR =
"a-zA-Z0-9\u00A0-\uD7FF\uF900-\uFDCF\uFDF0-\uFFEF";
/**
* Regular expression to match all IANA top-level domains for WEB_URL.
* List accurate as of 2011/01/12. List taken from:
* http://data.iana.org/TLD/tlds-alpha-by-domain.txt
* This pattern is auto-generated by frameworks/base/common/tools/make-iana-tld-pattern.py
*/
private static final String TOP_LEVEL_DOMAIN_STR_FOR_WEB_URL =
"(?:"
+ "(?:aaa|aarp|abarth|abb|abbott|abbvie|abc|able|abogado|abudhabi|academy|accenture|accountant|accountants|aco|active|actor|adac|ads|adult|aeg|aero|aetna|afamilycompany|afl|agakhan|agency|aig|aigo|airbus|airforce|airtel|akdn|alfaromeo|alibaba|alipay|allfinanz|allstate|ally|alsace|alstom|americanexpress|americanfamily|amex|amfam|amica|amsterdam|analytics|android|anquan|anz|apartments|app|apple|aquarelle|aramco|archi|army|arpa|art|arte|asda|asia|associates|athleta|attorney|auction|audi|audible|audio|auspost|author|auto|autos|avianca|aws|axa|azure|a[cdefgilmoqrstuwxz])"
+ "|(?:baby|baidu|banamex|bananarepublic|band|bank|bar|barcelona|barclaycard|barclays|barefoot|bargains|bauhaus|bayern|bbc|bbt|bbva|bcg|bcn|beats|beauty|beer|bentley|berlin|best|bestbuy|bet|bharti|bible|bid|bike|bing|bingo|bio|biz|black|blackfriday|blanco|blockbuster|blog|bloomberg|blue|bms|bmw|bnl|bnpparibas|boats|boehringer|bofa|bom|bond|boo|book|booking|boots|bosch|bostik|bot|boutique|bradesco|bridgestone|broadway|broker|brother|brussels|budapest|bugatti|build|builders|business|buy|buzz|bzh|b[abdefghijmnorstvwyz])"
+ "|(?:cab|cafe|cal|call|calvinklein|cam|camera|camp|cancerresearch|canon|capetown|capital|capitalone|car|caravan|cards|care|career|careers|cars|cartier|casa|cash|casino|cat|catering|cba|cbn|cbre|cbs|ceb|center|ceo|cern|cfa|cfd|chanel|channel|chase|chat|cheap|chintai|chloe|christmas|chrome|chrysler|church|cipriani|circle|cisco|citadel|citi|citic|city|cityeats|claims|cleaning|click|clinic|clinique|clothing|cloud|club|clubmed|coach|codes|coffee|college|cologne|com|comcast|commbank|community|company|compare|computer|comsec|condos|construction|consulting|contact|contractors|cooking|cookingchannel|cool|coop|corsica|country|coupon|coupons|courses|credit|creditcard|creditunion|cricket|crown|crs|cruises|csc|cuisinella|cymru|cyou|c[acdfghiklmnoruvwxyz])"
+ "|(?:dabur|dad|dance|date|dating|datsun|day|dclk|dds|deal|dealer|deals|degree|delivery|dell|deloitte|delta|democrat|dental|dentist|desi|design|dev|dhl|diamonds|diet|digital|direct|directory|discount|discover|dish|diy|dnp|docs|doctor|dodge|dog|doha|domains|dot|download|drive|dtv|dubai|duck|dunlop|duns|dupont|durban|dvag|d[ejkmoz])"
+ "|(?:earth|eat|eco|edeka|edu|education|email|emerck|energy|engineer|engineering|enterprises|epost|epson|equipment|ericsson|erni|esq|estate|esurance|eurovision|eus|events|everbank|exchange|expert|exposed|express|extraspace|e[cegrstu])"
+ "|(?:fage|fail|fairwinds|faith|family|fan|fans|farm|farmers|fashion|fast|fedex|feedback|ferrari|ferrero|fiat|fidelity|film|final|finance|financial|fire|firestone|firmdale|fish|fishing|fit|fitness|flickr|flights|flir|florist|flowers|fly|foo|foodnetwork|football|ford|forex|forsale|forum|foundation|fox|fresenius|frl|frogans|frontdoor|frontier|ftr|fujitsu|fujixerox|fund|furniture|futbol|fyi|f[ijkmor])"
+ "|(?:gal|gallery|gallo|gallup|game|games|gap|garden|gbiz|gdn|gea|gent|genting|george|ggee|gift|gifts|gives|giving|glade|glass|gle|global|globo|gmail|gmbh|gmo|gmx|godaddy|gold|goldpoint|golf|goo|goodhands|goodyear|goog|google|gop|got|gov|grainger|graphics|gratis|green|gripe|group|guardian|gucci|guge|guide|guitars|guru|g[abdefghilmnpqrstuwy])"
+ "|(?:hamburg|hangout|haus|hbo|hdfc|hdfcbank|health|healthcare|help|helsinki|here|hermes|hgtv|hiphop|hisamitsu|hitachi|hiv|hkt|hockey|holdings|holiday|homedepot|homegoods|homes|homesense|honda|honeywell|horse|host|hosting|hot|hoteles|hotmail|house|how|hsbc|htc|hughes|hyatt|hyundai|h[kmnrtu])"
+ "|(?:ibm|icbc|ice|icu|ieee|ifm|iinet|ikano|imamat|imdb|immo|immobilien|industries|infiniti|info|ing|ink|institute|insurance|insure|int|intel|international|intuit|investments|ipiranga|irish|iselect|ismaili|ist|istanbul|itau|itv|iwc|i[delmnoqrst])"
+ "|(?:jaguar|java|jcb|jcp|jeep|jetzt|jewelry|jlc|jll|jmp|jnj|jobs|joburg|jot|joy|jpmorgan|jprs|juegos|juniper|j[emop])"
+ "|(?:kaufen|kddi|kerryhotels|kerrylogistics|kerryproperties|kfh|kia|kim|kinder|kindle|kitchen|kiwi|koeln|komatsu|kosher|kpmg|kpn|krd|kred|kuokgroup|kyoto|k[eghimnprwyz])"
+ "|(?:lacaixa|ladbrokes|lamborghini|lamer|lancaster|lancia|lancome|land|landrover|lanxess|lasalle|lat|latino|latrobe|law|lawyer|lds|lease|leclerc|lefrak|legal|lego|lexus|lgbt|liaison|lidl|life|lifeinsurance|lifestyle|lighting|like|lilly|limited|limo|lincoln|linde|link|lipsy|live|living|lixil|loan|loans|locker|locus|loft|lol|london|lotte|lotto|love|lpl|lplfinancial|ltd|ltda|lundbeck|lupin|luxe|luxury|l[abcikrstuvy])"
+ "|(?:macys|madrid|maif|maison|makeup|man|management|mango|market|marketing|markets|marriott|marshalls|maserati|mattel|mba|mcd|mcdonalds|mckinsey|med|media|meet|melbourne|meme|memorial|men|menu|meo|metlife|miami|microsoft|mil|mini|mint|mit|mitsubishi|mlb|mls|mma|mobi|mobily|moda|moe|moi|mom|monash|money|monster|montblanc|mopar|mormon|mortgage|moscow|motorcycles|mov|movie|movistar|msd|mtn|mtpc|mtr|museum|mutual|mutuelle|m[acdeghklmnopqrstuvwxyz])"
+ "|(?:nab|nadex|nagoya|name|nationwide|natura|navy|nba|nec|net|netbank|netflix|network|neustar|new|news|next|nextdirect|nexus|nfl|ngo|nhk|nico|nike|nikon|ninja|nissan|nissay|nokia|northwesternmutual|norton|now|nowruz|nowtv|nra|nrw|ntt|nyc|n[acefgilopruz])"
+ "|(?:obi|off|office|okinawa|olayan|olayangroup|oldnavy|ollo|omega|one|ong|onl|online|onyourside|ooo|open|oracle|orange|org|organic|orientexpress|origins|osaka|otsuka|ott|ovh|om)"
+ "|(?:page|pamperedchef|panasonic|panerai|paris|pars|partners|parts|party|passagens|pay|pccw|pet|pfizer|pharmacy|philips|photo|photography|photos|physio|piaget|pics|pictet|pictures|pid|pin|ping|pink|pioneer|pizza|place|play|playstation|plumbing|plus|pnc|pohl|poker|politie|porn|post|pramerica|praxi|press|prime|pro|prod|productions|prof|progressive|promo|properties|property|protection|pru|prudential|pub|pwc|p[aefghklmnrstwy])"
+ "|(?:qpon|quebec|quest|qvc|qa)"
+ "|(?:racing|raid|read|realestate|realtor|realty|recipes|red|redstone|redumbrella|rehab|reise|reisen|reit|ren|rent|rentals|repair|report|republican|rest|restaurant|review|reviews|rexroth|rich|richardli|ricoh|rightathome|rio|rip|rocher|rocks|rodeo|room|rsvp|ruhr|run|rwe|ryukyu|r[eosuw])"
+ "|(?:saarland|safe|safety|sakura|sale|salon|samsclub|samsung|sandvik|sandvikcoromant|sanofi|sap|sapo|sarl|sas|save|saxo|sbi|sbs|sca|scb|schaeffler|schmidt|scholarships|school|schule|schwarz|science|scjohnson|scor|scot|seat|secure|security|seek|select|sener|services|ses|seven|sew|sex|sexy|sfr|shangrila|sharp|shaw|shell|shia|shiksha|shoes|shop|shopping|shouji|show|showtime|shriram|silk|sina|singles|site|ski|skin|sky|skype|sling|smart|smile|sncf|soccer|social|softbank|software|sohu|solar|solutions|song|sony|soy|space|spiegel|spot|spreadbetting|srl|srt|stada|staples|star|starhub|statebank|statefarm|statoil|stc|stcgroup|stockholm|storage|store|stream|studio|study|style|sucks|supplies|supply|support|surf|surgery|suzuki|swatch|swiftcover|swiss|sydney|symantec|systems|s[abcdeghijklmnortuvxyz])"
+ "|(?:tab|taipei|talk|taobao|target|tatamotors|tatar|tattoo|tax|taxi|tci|tdk|team|tech|technology|tel|telecity|telefonica|temasek|tennis|teva|thd|theater|theatre|tiaa|tickets|tienda|tiffany|tips|tires|tirol|tjmaxx|tjx|tkmaxx|tmall|today|tokyo|tools|top|toray|toshiba|total|tours|town|toyota|toys|trade|trading|training|travel|travelchannel|travelers|travelersinsurance|trust|trv|tube|tui|tunes|tushu|tvs|t[cdfghjklmnortvwz])"
+ "|(?:ubank|ubs|uconnect|unicom|university|uno|uol|ups|u[agksyz])"
+ "|(?:vacations|vana|vanguard|vegas|ventures|verisign|versicherung|vet|viajes|video|vig|viking|villas|vin|vip|virgin|visa|vision|vista|vistaprint|viva|vivo|vlaanderen|vodka|volkswagen|vote|voting|voto|voyage|vuelos|v[aceginu])"
+ "|(?:wales|walmart|walter|wang|wanggou|warman|watch|watches|weather|weatherchannel|webcam|weber|website|wed|wedding|weibo|weir|whoswho|wien|wiki|williamhill|win|windows|wine|winners|wme|wolterskluwer|woodside|work|works|world|wtc|wtf|w[fs])"
+ "|(?:xbox|xerox|xfinity|xihuan|xin|xn\\-\\-11b4c3d|xn\\-\\-1ck2e1b|xn\\-\\-1qqw23a|xn\\-\\-30rr7y|xn\\-\\-3bst00m|xn\\-\\-3ds443g|xn\\-\\-3e0b707e|xn\\-\\-3oq18vl8pn36a|xn\\-\\-3pxu8k|xn\\-\\-42c2d9a|xn\\-\\-45brj9c|xn\\-\\-45q11c|xn\\-\\-4gbrim|xn\\-\\-55qw42g|xn\\-\\-55qx5d|xn\\-\\-5su34j936bgsg|xn\\-\\-5tzm5g|xn\\-\\-6frz82g|xn\\-\\-6qq986b3xl|xn\\-\\-80adxhks|xn\\-\\-80ao21a|xn\\-\\-80asehdb|xn\\-\\-80aswg|xn\\-\\-8y0a063a|xn\\-\\-90a3ac|xn\\-\\-90ae|xn\\-\\-90ais|xn\\-\\-9dbq2a|xn\\-\\-9et52u|xn\\-\\-9krt00a|xn\\-\\-b4w605ferd|xn\\-\\-bck1b9a5dre4c|xn\\-\\-c1avg|xn\\-\\-c2br7g|xn\\-\\-cck2b3b|xn\\-\\-cg4bki|xn\\-\\-clchc0ea0b2g2a9gcd|xn\\-\\-czr694b|xn\\-\\-czrs0t|xn\\-\\-czru2d|xn\\-\\-d1acj3b|xn\\-\\-d1alf|xn\\-\\-e1a4c|xn\\-\\-eckvdtc9d|xn\\-\\-efvy88h|xn\\-\\-estv75g|xn\\-\\-fct429k|xn\\-\\-fhbei|xn\\-\\-fiq228c5hs|xn\\-\\-fiq64b|xn\\-\\-fiqs8s|xn\\-\\-fiqz9s|xn\\-\\-fjq720a|xn\\-\\-flw351e|xn\\-\\-fpcrj9c3d|xn\\-\\-fzc2c9e2c|xn\\-\\-fzys8d69uvgm|xn\\-\\-g2xx48c|xn\\-\\-gckr3f0f|xn\\-\\-gecrj9c|xn\\-\\-h2brj9c|xn\\-\\-hxt814e|xn\\-\\-i1b6b1a6a2e|xn\\-\\-imr513n|xn\\-\\-io0a7i|xn\\-\\-j1aef|xn\\-\\-j1amh|xn\\-\\-j6w193g|xn\\-\\-jlq61u9w7b|xn\\-\\-jvr189m|xn\\-\\-kcrx77d1x4a|xn\\-\\-kprw13d|xn\\-\\-kpry57d|xn\\-\\-kpu716f|xn\\-\\-kput3i|xn\\-\\-l1acc|xn\\-\\-lgbbat1ad8j|xn\\-\\-mgb9awbf|xn\\-\\-mgba3a3ejt|xn\\-\\-mgba3a4f16a|xn\\-\\-mgba7c0bbn0a|xn\\-\\-mgbaam7a8h|xn\\-\\-mgbab2bd|xn\\-\\-mgbayh7gpa|xn\\-\\-mgbb9fbpob|xn\\-\\-mgbbh1a71e|xn\\-\\-mgbc0a9azcg|xn\\-\\-mgbca7dzdo|xn\\-\\-mgberp4a5d4ar|xn\\-\\-mgbpl2fh|xn\\-\\-mgbt3dhd|xn\\-\\-mgbtx2b|xn\\-\\-mgbx4cd0ab|xn\\-\\-mix891f|xn\\-\\-mk1bu44c|xn\\-\\-mxtq1m|xn\\-\\-ngbc5azd|xn\\-\\-ngbe9e0a|xn\\-\\-node|xn\\-\\-nqv7f|xn\\-\\-nqv7fs00ema|xn\\-\\-nyqy26a|xn\\-\\-o3cw4h|xn\\-\\-ogbpf8fl|xn\\-\\-p1acf|xn\\-\\-p1ai|xn\\-\\-pbt977c|xn\\-\\-pgbs0dh|xn\\-\\-pssy2u|xn\\-\\-q9jyb4c|xn\\-\\-qcka1pmc|xn\\-\\-qxam|xn\\-\\-rhqv96g|xn\\-\\-rovu88b|xn\\-\\-s9brj9c|xn\\-\\-ses554g|xn\\-\\-t60b56a|xn\\-\\-tckwe|xn\\-\\-unup4y|xn\\-\\-vermgensberater\\-ctb|xn\\-\\-vermgensberatung\\-pwb|xn\\-\\-vhquv|xn\\-\\-vuq861b|xn\\-\\-w4r85el8fhu5dnra|xn\\-\\-w4rs40l|xn\\-\\-wgbh1c|xn\\-\\-wgbl6a|xn\\-\\-xhq521b|xn\\-\\-xkc2al3hye2a|xn\\-\\-xkc2dl3a5ee0h|xn\\-\\-y9a3aq|xn\\-\\-yfro4i67o|xn\\-\\-ygbi2ammx|xn\\-\\-zfr164b|xperia|xxx|xyz)"
+ "|(?:yachts|yahoo|yamaxun|yandex|yodobashi|yoga|yokohama|you|youtube|yun|y[et])"
+ "|(?:zappos|zara|zero|zip|zippo|zone|zuerich|z[amw])))";
private static final String BITCOIN_URI_PATTERN =
"bitcoin:[1-9a-km-zA-HJ-NP-Z]{27,34}(\\?[a-zA-Z0-9$\\-_.+!*'(),%:@&=]*)?";
/**
* Regular expression pattern to match most part of RFC 3987
* Internationalized URLs, aka IRIs. Commonly used Unicode characters are
* added.
*/
private static final Pattern WEB_URL_PATTERN = Pattern.compile(
"((?:(http|https|Http|Https|rtsp|Rtsp):\\/\\/(?:(?:[a-zA-Z0-9\\$\\-\\_\\.\\+\\!\\*\\'\\(\\)"
+ "\\,\\;\\?\\&\\=]|(?:\\%[a-fA-F0-9]{2})){1,64}(?:\\:(?:[a-zA-Z0-9\\$\\-\\_"
+ "\\.\\+\\!\\*\\'\\(\\)\\,\\;\\?\\&\\=]|(?:\\%[a-fA-F0-9]{2})){1,25})?\\@)?)?"
+ "((?:(?:[" + GOOD_IRI_CHAR + "][" + GOOD_IRI_CHAR + "\\-]{0,64}\\.)+" // named host
+ TOP_LEVEL_DOMAIN_STR_FOR_WEB_URL
+ "|(?:(?:25[0-5]|2[0-4]" // or ip address
+ "[0-9]|[0-1][0-9]{2}|[1-9][0-9]|[1-9])\\.(?:25[0-5]|2[0-4][0-9]"
+ "|[0-1][0-9]{2}|[1-9][0-9]|[1-9]|0)\\.(?:25[0-5]|2[0-4][0-9]|[0-1]"
+ "[0-9]{2}|[1-9][0-9]|[1-9]|0)\\.(?:25[0-5]|2[0-4][0-9]|[0-1][0-9]{2}"
+ "|[1-9][0-9]|[0-9])))"
+ "(?:\\:\\d{1,5})?)" // plus option port number
+ "(\\/(?:(?:[" + GOOD_IRI_CHAR + "\\;\\/\\?\\:\\@\\&\\=\\#\\~" // plus option query params
+ "\\-\\.\\+\\!\\*\\'\\(\\)\\,\\_])|(?:\\%[a-fA-F0-9]{2}))*)?"
+ "(?:\\b|$)"); // and finally, a word boundary or end of
/**
* When generating previews, Spannable objects that can't be converted into a String are
* represented as 0xfffc. When displayed, these show up as undisplayed squares. These constants
@ -339,7 +284,7 @@ public class HtmlConverter {
StringBuffer sb = new StringBuffer(text.length() + TEXT_TO_HTML_EXTRA_BUFFER_LENGTH);
sb.append(htmlifyMessageHeader());
linkifyText(text, sb);
UriLinkifier.linkifyText(text, sb);
sb.append(htmlifyMessageFooter());
text = sb.toString();
@ -428,32 +373,6 @@ public class HtmlConverter {
}
}
/**
* Searches for link-like text in a string and turn it into a link. Append the result to
* <tt>outputBuffer</tt>. <tt>text</tt> is not modified.
* @param text Plain text to be linkified.
* @param outputBuffer Buffer to append linked text to.
*/
protected static void linkifyText(final String text, final StringBuffer outputBuffer) {
String prepared = text.replaceAll(BITCOIN_URI_PATTERN, "<a href=\"$0\">$0</a>");
Matcher m = WEB_URL_PATTERN.matcher(prepared);
while (m.find()) {
int start = m.start();
if (start == 0 || (start != 0 && prepared.charAt(start - 1) != '@')) {
if (m.group().indexOf(':') > 0) { // With no URI-schema we may get "http:/" links with the second / missing
m.appendReplacement(outputBuffer, "<a href=\"$0\">$0</a>");
} else {
m.appendReplacement(outputBuffer, "<a href=\"http://$0\">$0</a>");
}
} else {
m.appendReplacement(outputBuffer, "$0");
}
}
m.appendTail(outputBuffer);
}
/*
* Lightweight method to check whether the message contains emoji or not.
* Useful to avoid calling the heavyweight convertEmoji2Img method.
@ -1384,7 +1303,7 @@ public class HtmlConverter {
// Linkify the message.
StringBuffer linkified = new StringBuffer(htmlified.length() + TEXT_TO_HTML_EXTRA_BUFFER_LENGTH);
linkifyText(htmlified, linkified);
UriLinkifier.linkifyText(htmlified, linkified);
// Add newlines and unescaping.
//

View file

@ -0,0 +1,316 @@
package com.fsck.k9.message.html;
import java.net.IDN;
import java.util.Locale;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* Parses and "linkifies" http links.
* <p>
* This class is in parts inspired by OkHttp's HttpUrl (https://github.com/square/okhttp/blob/master/okhttp/src/main/java/okhttp3/HttpUrl.java),s
* but leaving out much of the parsing part.
*/
class HttpUriParser implements UriParser {
// This string represent character group sub-delim as described in RFC 3986
private static final String SUB_DELIM = "!$&'()*+,;=";
private static final Pattern IPv4_PATTERN =
Pattern.compile("(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})\\.(\\d{1,3})(:(\\d{0,5}))?");
@Override
public int linkifyUri(String text, int startPos, StringBuffer outputBuffer) {
int currentPos = startPos;
// Test scheme
String shortScheme = text.substring(currentPos, Math.min(currentPos + 7, text.length()));
String longScheme = text.substring(currentPos, Math.min(currentPos + 8, text.length()));
if (shortScheme.equalsIgnoreCase("https://")) {
currentPos += "https://".length();
} else if (shortScheme.equalsIgnoreCase("http://")) {
currentPos += "http://".length();
} else if (longScheme.equalsIgnoreCase("rtsp://")) {
currentPos += "rtsp://".length();
} else {
// Unsupported scheme
return startPos;
}
// Test authority
int authorityEnd = text.indexOf('/', currentPos);
if (authorityEnd == -1) {
authorityEnd = text.length();
}
// Authority: Take a look at user info if available
currentPos = matchUserInfoIfAvailable(text, currentPos, authorityEnd);
// Authority: Take a look at host
if (!tryMatchDomainName(text, currentPos, authorityEnd) &&
!tryMatchIpv4Address(text, currentPos, authorityEnd, true) &&
!tryMatchIpv6Address(text, currentPos, authorityEnd)) {
return startPos;
}
currentPos = authorityEnd;
// Test path
if (currentPos < text.length() && text.charAt(currentPos) == '/') {
currentPos = matchUnreservedPCTEncodedSubDelimClassesGreedy(text, currentPos + 1, "/:@");
}
// Test for query
if (currentPos < text.length() && text.charAt(currentPos) == '?') {
currentPos = matchUnreservedPCTEncodedSubDelimClassesGreedy(text, currentPos + 1, ":@/?");
}
// Test for fragment.
if (currentPos < text.length() && text.charAt(currentPos) == '#') {
currentPos = matchUnreservedPCTEncodedSubDelimClassesGreedy(text, currentPos + 1, ":@/?");
}
// Final link generation
String linkifiedUri = String.format("<a href=\"%1$s\">%1$s</a>", text.substring(startPos, currentPos));
outputBuffer.append(linkifiedUri);
return currentPos;
}
private int matchUserInfoIfAvailable(String text, int startPos, int authorityEnd) {
int userInfoEnd = text.indexOf('@', startPos);
if (userInfoEnd != -1 && userInfoEnd < authorityEnd) {
if (matchUnreservedPCTEncodedSubDelimClassesGreedy(text, startPos, ":") != userInfoEnd) {
// Illegal character in user info
return startPos;
}
return userInfoEnd + 1;
}
return startPos;
}
private boolean tryMatchDomainName(String text, int startPos, int authorityEnd) {
// Partly from OkHttp's HttpUrl (https://github.com/square/okhttp/blob/master/okhttp/src/main/java/okhttp3/HttpUrl.java)
try {
// Check for port
int portPos = text.indexOf(':', startPos);
boolean hasPort = portPos != -1 && portPos < authorityEnd;
if (hasPort) {
int port = 0;
for (int i = portPos + 1; i < authorityEnd; i++) {
int c = text.codePointAt(i);
if (c < '0' || c > '9') {
return false;
}
port = port * 10 + c - '0';
}
if (port > 65535) {
return false;
}
}
// Check actual domain
String result = IDN.toASCII(text.substring(startPos, authorityEnd)).toLowerCase(Locale.US);
if (result.isEmpty()) {
return false;
}
// Confirm that the IDN ToASCII result doesn't contain any illegal characters.
for (int i = 0; i < result.length(); i++) {
char c = result.charAt(i);
// The WHATWG Host parsing rules accepts some character codes which are invalid by
// definition for OkHttp's host header checks (and the WHATWG Host syntax definition). Here
// we rule out characters that would cause problems in host headers.
if (c <= '\u001f' || c >= '\u007f') {
return false;
}
// Check for the characters mentioned in the WHATWG Host parsing spec:
// U+0000, U+0009, U+000A, U+000D, U+0020, "#", "%", "/", ":", "?", "@", "[", "\", and "]"
// (excluding the characters covered above).
if (" #%/:?@[\\]".indexOf(c) != -1) {
return false;
}
}
return true;
} catch (IllegalArgumentException e) {
return false;
}
}
private boolean tryMatchIpv4Address(String text, int startPos, int authorityEnd, boolean portAllowed) {
Matcher matcher = IPv4_PATTERN.matcher(text.subSequence(startPos, authorityEnd));
if (!matcher.matches()) {
return false;
}
// Validate segments
for (int i = 1; i <= 4; i++) {
int segment = Integer.parseInt(matcher.group(1));
if (segment > 255) {
return false;
}
}
// Make sure port does not exist if missing
if (!portAllowed && matcher.group(5) != null) {
return false;
}
// Validate optional port
String portString = matcher.group(6);
if (portString != null && !portString.isEmpty()) {
int port = Integer.parseInt(portString);
if (port > 65535) {
return false;
}
}
return true;
}
private boolean tryMatchIpv6Address(String text, int startPos, int authorityEnd) {
// General validation
if (text.codePointAt(startPos) != '[') {
return false;
}
int addressEnd = text.indexOf(']');
if (addressEnd == -1 || addressEnd >= authorityEnd) {
return false;
}
// Actual parsing
int currentPos = startPos + 1;
int beginSegmentsCount = 0;
int endSegmentsCount = 0;
// Handle :: separator and segments in front of it
int compressionPos = text.indexOf("::");
boolean compressionEnabled = compressionPos != -1 && compressionPos < addressEnd;
if (compressionEnabled) {
while (currentPos < compressionPos) {
// Check segment separator
if (beginSegmentsCount > 0) {
if (text.codePointAt(currentPos) != ':') {
return false;
} else {
++currentPos;
}
}
// Parse segment
int possibleSegmentEnd =
parse16BitHexSegment(text, currentPos, Math.min(currentPos + 4, compressionPos));
if (possibleSegmentEnd == currentPos) {
return false;
}
currentPos = possibleSegmentEnd;
++beginSegmentsCount;
}
currentPos += 2; // Skip :: separator
}
// Parse end segments
while (currentPos < addressEnd && (beginSegmentsCount + endSegmentsCount) < 8) {
// Check segment separator
if (endSegmentsCount > 0) {
if (text.codePointAt(currentPos) != ':') {
return false;
} else {
++currentPos;
}
}
// Small look ahead, do not run into IPv4 tail (7 is IPv4 minimum length)
int nextColon = text.indexOf(':', currentPos);
if ((nextColon == -1 || nextColon > addressEnd) && (addressEnd - currentPos) >= 7) {
break;
}
// Parse segment
int possibleSegmentEnd = parse16BitHexSegment(text, currentPos, Math.min(currentPos + 4, addressEnd));
if (possibleSegmentEnd == currentPos) {
return false;
}
currentPos = possibleSegmentEnd;
++endSegmentsCount;
}
// We have 3 valid cases here
if (currentPos == addressEnd) {
// 1) No compression and full address, everything fine
// 2) Compression enabled and whole address parsed, everything fine as well
if ((!compressionEnabled && beginSegmentsCount + endSegmentsCount == 8) ||
(compressionEnabled && beginSegmentsCount + endSegmentsCount < 8)) {
// Only optional port left, skip address bracket
++currentPos;
} else {
return false;
}
} else {
// 3) Still some stuff missing, check for IPv4 as tail necessary
if (!tryMatchIpv4Address(text, currentPos, addressEnd, false)) {
return false;
}
currentPos = addressEnd + 1;
}
// Check optional port
if (currentPos == authorityEnd) {
return true;
}
if (text.codePointAt(currentPos) != ':' || currentPos + 1 == authorityEnd) {
return false;
}
++currentPos;
int port = 0;
for (int i = currentPos; i < authorityEnd; i++) {
int c = text.codePointAt(i);
if (c < '0' || c > '9') {
return false;
}
port = port * 10 + c - '0';
}
return port <= 65535;
}
private int parse16BitHexSegment(String text, int startPos, int endPos) {
int currentPos = startPos;
while (isHexDigit(text.codePointAt(currentPos)) && currentPos < endPos) {
++currentPos;
}
return currentPos;
}
private int matchUnreservedPCTEncodedSubDelimClassesGreedy(String text, int startPos, String additionalCharacters) {
String allowedCharacters = SUB_DELIM + "-._~" + additionalCharacters;
int currentPos;
int shouldBeHex = 0;
for (currentPos = startPos; currentPos < text.length(); currentPos++) {
int c = text.codePointAt(currentPos);
if (isHexDigit(c)) {
shouldBeHex = Math.max(shouldBeHex - 1, 0);
} else if (shouldBeHex == 0) {
if (allowedCharacters.indexOf(c) != -1) {
// Everything ok here :)
} else if (c == '%') {
shouldBeHex = 2;
} else {
break;
}
} else {
break;
}
}
return currentPos;
}
private boolean isHexDigit(int c) {
return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9');
}
}

View file

@ -0,0 +1,81 @@
package com.fsck.k9.message.html;
import java.util.HashMap;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import android.text.TextUtils;
/**
* Allows conversion of link in text to html link.
*/
public class UriLinkifier {
/**
* Regular expression pattern to match uri scheme and parsers for supported uris as defined in RFC 3987
*/
private static final Pattern URI_SCHEME;
private static final Map<String, UriParser> SUPPORTED_URIS;
private static final String SCHEME_SEPARATOR = " (";
static {
SUPPORTED_URIS = new HashMap<>();
SUPPORTED_URIS.put("bitcoin:", new BitcoinUriParser());
UriParser httpParser = new HttpUriParser();
SUPPORTED_URIS.put("http:", httpParser);
SUPPORTED_URIS.put("https:", httpParser);
SUPPORTED_URIS.put("rtsp:", httpParser);
String allSchemes = TextUtils.join("|", SUPPORTED_URIS.keySet());
URI_SCHEME = Pattern.compile(allSchemes, Pattern.CASE_INSENSITIVE);
}
/**
* Searches for link-like text in a string and turn it into a link. Append the result to
* <tt>outputBuffer</tt>. <tt>text</tt> is not modified.
*
* @param text
* Plain text to be linkified.
* @param outputBuffer
* Buffer to append linked text to.
*/
public static void linkifyText(final String text, final StringBuffer outputBuffer) {
int currentPos = 0;
Matcher matcher = URI_SCHEME.matcher(text);
while (matcher.find(currentPos)) {
int startPos = matcher.start();
String textBeforeMatch = text.substring(currentPos, startPos);
outputBuffer.append(textBeforeMatch);
if (!textBeforeMatch.isEmpty() &&
!SCHEME_SEPARATOR.contains(textBeforeMatch.substring(textBeforeMatch.length() - 1))) {
outputBuffer.append(text.charAt(startPos));
currentPos = startPos + 1;
continue;
}
// Find responsible parser and let it do it's job
String scheme = matcher.group();
UriParser parser = SUPPORTED_URIS.get(scheme.toLowerCase());
int newPos = parser.linkifyUri(text, startPos, outputBuffer);
// Handle invalid uri, at least advance by one to prevent endless loop
if (newPos <= startPos) {
outputBuffer.append(text.charAt(startPos));
currentPos++;
} else {
currentPos = (newPos > currentPos) ? newPos : currentPos + 1;
}
if (currentPos >= text.length()) {
break;
}
}
// Copy rest
outputBuffer.append(text.substring(currentPos));
}
}

View file

@ -0,0 +1,15 @@
package com.fsck.k9.message.html;
/**
* General framework to handle uris when parsing. Allows different handling depending on the scheme identifier.
*/
public interface UriParser {
/**
* Parse and linkify scheme specific uri beginning from given position. The result will be written to given buffer.
* @param text String to parse uri from.
* @param startPos Position where uri starts (first letter of scheme).
* @param outputBuffer Buffer where linkified variant of uri is written to.
* @return Index where parsed uri ends (first non-uri letter). Should be startPos or smaller if no valid uri was found.
*/
int linkifyUri(String text, int startPos, StringBuffer outputBuffer);
}

View file

@ -193,21 +193,6 @@ public class HtmlConverterTest {
+ "</pre>", result);
}
@Test
public void testLinkifyBitcoinAndHttpUri() {
String text = "bitcoin:19W6QZkx8SYPG7BBCS7odmWGRxqRph5jFU http://example.com/";
StringBuffer outputBuffer = new StringBuffer();
HtmlConverter.linkifyText(text, outputBuffer);
assertEquals("<a href=\"bitcoin:19W6QZkx8SYPG7BBCS7odmWGRxqRph5jFU\">" +
"bitcoin:19W6QZkx8SYPG7BBCS7odmWGRxqRph5jFU" +
"</a> " +
"<a href=\"http://example.com/\">" +
"http://example.com/" +
"</a>", outputBuffer.toString());
}
@Test
public void issue2259Spec() {
String text = "text\n" +

View file

@ -0,0 +1,216 @@
package com.fsck.k9.message.html;
import com.fsck.k9.K9RobolectricTestRunner;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.annotation.Config;
import static junit.framework.Assert.assertEquals;
@RunWith(K9RobolectricTestRunner.class)
@Config(manifest = Config.NONE)
public class HttpUriParserTest {
private HttpUriParser parser;
private StringBuffer outputBuffer;
@Before
public void setUp() {
parser = new HttpUriParser();
outputBuffer = new StringBuffer();
}
@Test
public void testSimpleDomain() {
String text = "http://www.google.com";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testDomainWithTrailingSlash() {
String text = "http://www.google.com/";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testDomainWithoutWWW() {
String text = "http://google.com/";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testDomainWithTrailingSpace() {
String text = "http://google.com/ ";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals("<a href=\"http://google.com/\">http://google.com/</a>", outputBuffer.toString());
assertEquals(text.length() - 1, endPos);
}
@Test
public void testDomainWithTrailingSpaceNewline() {
String text = "http://google.com/ \n";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals("<a href=\"http://google.com/\">http://google.com/</a>", outputBuffer.toString());
assertEquals(text.length() - 2, endPos);
}
@Test
public void testDomainWithTrailingNewline() {
String text = "http://google.com/\n";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals("<a href=\"http://google.com/\">http://google.com/</a>", outputBuffer.toString());
assertEquals(text.length() - 1, endPos);
}
@Test
public void testDomainsWithQueryAndFragment() {
String text = "http://google.com/give/me/?q=mode&c=information#only-the-best";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(
"<a href=\"http://google.com/give/me/?q=mode&c=information#only-the-best\">http://google.com/give/me/?q=mode&c=information#only-the-best</a>",
outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testDomainsWithQuery() {
String text = "http://google.com/give/me/?q=mode&c=information";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(
"<a href=\"http://google.com/give/me/?q=mode&c=information\">http://google.com/give/me/?q=mode&c=information</a>",
outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testDomainsWithFragment() {
String text = "http://google.com/give/me#only-the-best";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(
"<a href=\"http://google.com/give/me#only-the-best\">http://google.com/give/me#only-the-best</a>",
outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testDomainsWithQueryAndFragmentWithoutWWWW() {
String text = "http://google.com/give/me/?q=mode+c=information#only-the-best\n";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(
"<a href=\"http://google.com/give/me/?q=mode+c=information#only-the-best\">http://google.com/give/me/?q=mode+c=information#only-the-best</a>",
outputBuffer.toString());
assertEquals(text.length() - 1, endPos);
}
@Test
public void testIpv4Address() {
String text = "http://127.0.0.1";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv4AddressWithTrailingSlash() {
String text = "http://127.0.0.1/";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv4AddressWithEmptyPort() {
String text = "http://127.0.0.1:";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv4AddressWithPort() {
String text = "http://127.0.0.1:524/";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv6Address() {
String text = "http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv6AddressWithPort() {
String text = "http://[FEDC:BA98:7654:3210:FEDC:BA98:7654:3210]:80";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv6AddressShort() {
String text = "http://[1080:0:0:0:8:800:200C:417A]/";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv6AddressWithEndCompression() {
String text = "http://[3ffe:2a00:100:7031::1]";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv6AddressWithBeginCompression() {
String text = "http://[1080::8:800:200C:417A]/";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv6AddressWithPrependedCompression() {
String text = "http://[::192.9.5.5]/";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv6AddressWithCompressionPort() {
String text = "http://[::FFFF:129.144.52.38]:80/";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv6AddressWithTrailingIp4() {
String text = "http://[::192.9.5.5]/";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
@Test
public void testIpv6AddressWithTrailingIp4AndPort() {
String text = "http://[::192.9.5.5]:80/";
int endPos = parser.linkifyUri(text, 0, outputBuffer);
assertEquals(String.format("<a href=\"%1$s\">%1$s</a>", text), outputBuffer.toString());
assertEquals(text.length(), endPos);
}
}

View file

@ -0,0 +1,95 @@
package com.fsck.k9.message.html;
import com.fsck.k9.K9RobolectricTestRunner;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.robolectric.annotation.Config;
import static junit.framework.Assert.assertEquals;
@RunWith(K9RobolectricTestRunner.class)
@Config(manifest = Config.NONE)
public class UriLinkifierTest {
@Test
public void testLinkifyBitcoinAndHttpUri() {
String text = "bitcoin:19W6QZkx8SYPG7BBCS7odmWGRxqRph5jFU http://example.com/";
StringBuffer outputBuffer = new StringBuffer();
UriLinkifier.linkifyText(text, outputBuffer);
assertEquals("<a href=\"bitcoin:19W6QZkx8SYPG7BBCS7odmWGRxqRph5jFU\">" +
"bitcoin:19W6QZkx8SYPG7BBCS7odmWGRxqRph5jFU" +
"</a> " +
"<a href=\"http://example.com/\">" +
"http://example.com/" +
"</a>", outputBuffer.toString());
}
@Test
public void testSimpleHttpUri() {
String text = "http://www.google.com";
StringBuffer outputBuffer = new StringBuffer();
UriLinkifier.linkifyText(text, outputBuffer);
assertEquals("<a href=\"http://www.google.com\">http://www.google.com</a>", outputBuffer.toString());
}
@Test
public void testHttpUriWithTrailingSlash() {
String text = "http://www.google.com/";
StringBuffer outputBuffer = new StringBuffer();
UriLinkifier.linkifyText(text, outputBuffer);
assertEquals("<a href=\"http://www.google.com/\">http://www.google.com/</a>",
outputBuffer.toString());
}
@Test
public void testHttpUriWithoutWWW() {
String text = "http://google.com/";
StringBuffer outputBuffer = new StringBuffer();
UriLinkifier.linkifyText(text, outputBuffer);
assertEquals("<a href=\"http://google.com/\">http://google.com/</a>", outputBuffer.toString());
}
@Test
public void testHttpUriWithTrailingSpace() {
String text = "http://google.com/ ";
StringBuffer outputBuffer = new StringBuffer();
UriLinkifier.linkifyText(text, outputBuffer);
assertEquals("<a href=\"http://google.com/\">http://google.com/</a> ", outputBuffer.toString());
}
@Test
public void testHttpUriWithTrailingSpaceNewline() {
String text = "http://google.com/ \n";
StringBuffer outputBuffer = new StringBuffer();
UriLinkifier.linkifyText(text, outputBuffer);
assertEquals("<a href=\"http://google.com/\">http://google.com/</a> \n",
outputBuffer.toString());
}
@Test
public void testHttpUriWithTrailingNewline() {
String text = "http://google.com/\n";
StringBuffer outputBuffer = new StringBuffer();
UriLinkifier.linkifyText(text, outputBuffer);
assertEquals("<a href=\"http://google.com/\">http://google.com/</a>\n", outputBuffer.toString());
}
@Test
public void testIgnorePartialHttpUriScheme() {
String text = "myhttp://example.org";
StringBuffer outputBuffer = new StringBuffer();
UriLinkifier.linkifyText(text, outputBuffer);
assertEquals(text, outputBuffer.toString());
}
@Test
public void testPartialHttpUriSchemeWithSeparator() {
String text = "(http://example.org";
StringBuffer outputBuffer = new StringBuffer();
UriLinkifier.linkifyText(text, outputBuffer);
assertEquals("(<a href=\"http://example.org\">http://example.org</a>", outputBuffer.toString());
}
}