diff --git a/src/main/java/org/openaudible/audible/AudibleScraper.java b/src/main/java/org/openaudible/audible/AudibleScraper.java index d03a8ef..451b080 100644 --- a/src/main/java/org/openaudible/audible/AudibleScraper.java +++ b/src/main/java/org/openaudible/audible/AudibleScraper.java @@ -19,13 +19,12 @@ import org.openaudible.books.BookElement; import org.openaudible.progress.IProgressTask; import org.openaudible.util.EventTimer; import org.openaudible.util.HTMLUtil; +import org.openaudible.util.Util; import org.w3c.dom.Node; -import org.w3c.dom.NodeList; import java.io.File; import java.io.IOException; import java.net.URL; -import java.net.URLDecoder; import java.util.*; // audible.com web page scraper @@ -513,12 +512,29 @@ public class AudibleScraper { if (next == null) { assert (pageNum == 1); setURL("/lib", "Reading Library..."); + + DomElement purchaseDateFilter = page.getElementByName("purchaseDateFilter"); + if (purchaseDateFilter!=null) + { + DomNodeList nodes = purchaseDateFilter.getChildNodes(); + for (DomNode n:nodes) + { + if (n instanceof HtmlOption) + { + // "all" is first option.. + ((HtmlOption) n).click(); + break ; + } + } + } } else { // getProgress().setTask("Getting a list of your library. ); EventTimer evt = new EventTimer(); String u = next.getAttribute("data-url"); if (u != null) { - if (!u.endsWith("&")) u += "&"; + + if (!u.endsWith("&")) + u += "&"; u += "page=" + pageNum; setURL(u, "Reading Library page " + pageNum + "... Found " + results.size() + " books"); } else { diff --git a/src/main/java/org/openaudible/audible/LibraryParser.java b/src/main/java/org/openaudible/audible/LibraryParser.java index 8392bfd..2db71b8 100644 --- a/src/main/java/org/openaudible/audible/LibraryParser.java +++ b/src/main/java/org/openaudible/audible/LibraryParser.java @@ -9,6 +9,7 @@ import org.openaudible.util.HTMLUtil; import org.openaudible.util.Util; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; // Parse a page of library inforation. @@ -20,6 +21,7 @@ public enum LibraryParser { boolean debug = false; + // Expected Columns: // Image // Title @@ -38,13 +40,11 @@ public enum LibraryParser { } // get product id and asin first. - static BookColumns [] parseOrder = {Download, Other, Image, Title, Author, Length, Date_Added, Ratings}; + static BookColumns[] parseOrder = {Download, Other, Image, Title, Author, Length, Date_Added, Ratings}; } - - // return "next" button for next page of results. public HtmlElement getNextPage(HtmlPage page) { HtmlElement next = null; @@ -57,6 +57,8 @@ public enum LibraryParser { next = a; } } + + return next; } @@ -74,9 +76,8 @@ public enum LibraryParser { int purchaseDateIndex = -1; List header = table.getElementsByTagName("th"); - if (header.size() != BookColumns.size()) - { - LOG.info("Skipping table with:"+header.size()+" cols"); + if (header.size() != BookColumns.size()) { + LOG.info("Skipping table with:" + header.size() + " cols"); return list; } @@ -124,7 +125,7 @@ public enum LibraryParser { if (r.getCells().size() != BookColumns.size()) { LOG.error("wrong number of columns found: " + r.getCells().size() + " != " + BookColumns.size()); LOG.error(xml); - if (debug) HTMLUtil.debugNode(r, "bad_col.xml"); + if (debug) HTMLUtil.debugNode(r, "bad_col.xml"); return null; } @@ -164,7 +165,7 @@ public enum LibraryParser { } int ch = text.indexOf("\n"); - if (ch!=-1) + if (ch != -1) text = text.substring(0, ch); @@ -180,28 +181,27 @@ public enum LibraryParser { // /pd/Fiction/Exodus-Audiobook/B008I3VMMQ? if (url.startsWith("/pd/")) { int q = url.indexOf("?"); - if (q!=-1) + if (q != -1) url = url.substring(0, q); boolean ok = false; if (b.has(BookElement.asin) && url.contains(b.getAsin())) - ok=true; + ok = true; if (b.has(BookElement.product_id) && url.contains(b.getProduct_id())) - ok=true; + ok = true; if (ok) b.setInfoLink(url); else - LOG.info("Unknown product link for "+b+" at "+url); + LOG.info("Unknown product link for " + b + " at " + url); } } - if (text.contains("by parts")) - { - LOG.error("error with title: "+text); - if (debug) HTMLUtil.debugNode(cell, col.name()+".xml"); + if (text.contains("by parts")) { + LOG.error("error with title: " + text); + if (debug) HTMLUtil.debugNode(cell, col.name() + ".xml"); // bug check. } @@ -241,29 +241,16 @@ public enum LibraryParser { cust_id=xxx&DownloadType=Now&transfer_player=1&title=Book Title&codec=LC_32_22050_Mono&awtype=AAX"; */ + private void parseDownloadURL(HtmlAnchor a, Book b) { String url = a.getHrefAttribute(); - String obj = a.toString(); - try { - - String args = url.substring(url.indexOf("?") + 1, url.length()); - - String split[] = args.split("&"); - for (String params : split) { - String kv[] = params.split("="); - if (kv.length == 2) { - // LOG.info(kv[0] + "=" + kv[1]); - BookElement elem = BookElement.findByName(kv[0]); - if (elem != null) { - b.set(elem, kv[1]); - } - } else { - LOG.error("bad url param:" + params + " for " + url); /// happens when there is an & in title. - } + HashMap args = Util.urlGetArgs(url); + for (String k : args.keySet()) { + BookElement elem = BookElement.findByName(k); + if (elem != null) { + b.set(elem, args.get(k)); } - } catch (Throwable th) { - LOG.error("Error parsing anchor:" + url + " for " + obj); } } diff --git a/src/main/java/org/openaudible/desktop/swt/manager/VersionCheck.java b/src/main/java/org/openaudible/desktop/swt/manager/VersionCheck.java index bfd297c..0707a6d 100644 --- a/src/main/java/org/openaudible/desktop/swt/manager/VersionCheck.java +++ b/src/main/java/org/openaudible/desktop/swt/manager/VersionCheck.java @@ -1,10 +1,10 @@ package org.openaudible.desktop.swt.manager; -import com.google.gson.JsonObject; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.eclipse.swt.SWT; import org.eclipse.swt.widgets.Shell; +import org.json.JSONObject; import org.openaudible.desktop.swt.gui.MessageBoxFactory; import org.openaudible.desktop.swt.manager.menu.CommandCenter; import org.openaudible.util.HTTPGet; @@ -20,15 +20,15 @@ public enum VersionCheck { // if verbose return state regardless. // if !verbose, only alert when new version is available. public void checkForUpdate(Shell shell, boolean verbose) { - JsonObject obj = versionCheck(); - String msg = obj.get("msg").getAsString(); - String title = obj.get("title").getAsString(); + JSONObject obj = versionCheck(); + String msg = obj.optString("msg"); + String title = obj.optString("title","Version Check"); - int diff = obj.get("diff").getAsInt(); + int diff = obj.optInt("diff", 0); if (diff < 0) { MessageBoxFactory.showGeneral(shell, SWT.ICON_INFORMATION, title, msg); if (obj.has("site")) { - String url = obj.get("site").getAsString(); + String url = obj.getString("site"); AudibleGUI.instance.browse(url); } @@ -43,14 +43,14 @@ public enum VersionCheck { // return msg; } - public JsonObject getVersion() throws IOException { + public JSONObject getVersion() throws IOException { String url = Version.versionLink; url += "?"; url += "platform=" + Platform.getPlatform().toString(); url += "&version=" + Version.appVersion; // url += "&count=" + audible.getBookCount(); LOG.info("versionCheck: " + url); - return HTTPGet.instance.getJSON(url); + return HTTPGet.instance.getJSON(url); } /** @@ -84,56 +84,54 @@ public enum VersionCheck { return Integer.signum(vals1.length - vals2.length); } - public JsonObject versionCheck() { - JsonObject obj = null; + public JSONObject versionCheck() { + JSONObject obj = null; try { obj = getVersion(); if (!obj.has("version")) throw new IOException("missing version field\n" + obj); - String releaseVersion = obj.get("version").getAsString(); + String releaseVersion = obj.getString("version"); int diff = versionCompare(Version.appVersion, releaseVersion); - obj.addProperty("diff", "" + diff); + obj.put("diff", "" + diff); String msg, title; if (diff < 0) { title = "Update Available"; msg = "An update is available!\nYour version: " + Version.appVersion + "\nRelease Version:" + releaseVersion; - if (obj.has("required") && obj.get("required").getAsBoolean()) + if (obj.optBoolean("required", false)) { msg += "\nThis upgrade is required. Old versions no longer supported."; CommandCenter.instance.expiredApp = true; } + msg += "\n"+obj.optString("old_news", ""); } else if (diff > 0) { title = "Using Pre-release"; msg = "You appear to be using a pre-release version\nYour version: " + Version.appVersion + "\nLatest Version:" + releaseVersion; + msg += "\n"+obj.optString("pre_release_news", ""); } else { title = "No update at this time"; msg = "Using the latest release version."; + msg += "\n"+obj.optString("current_news", ""); // allow a news field } - if (obj.has("news")) - { - msg +="\n"+obj.get("news").getAsString(); - } - if (obj.has("kill")) { - msg +="\n"+obj.get("kill").getAsString(); + msg +="\n"+obj.getString("kill"); CommandCenter.instance.expiredApp = true; } - obj.addProperty("msg", msg); - obj.addProperty("title", title); + obj.put("msg", msg); + obj.put("title", title); } catch (IOException e) { if (obj == null) - obj = new JsonObject(); - obj.addProperty("msg", "Error checking for latest version.\nError message: " + e.getMessage()); - obj.addProperty("title", "Version check failed"); + obj = new JSONObject(); + obj.put("msg", "Error checking for latest version.\nError message: " + e.getMessage()); + obj.put("title", "Version check failed"); } return obj; } diff --git a/src/main/java/org/openaudible/desktop/swt/util/shop/PaintShop.java b/src/main/java/org/openaudible/desktop/swt/util/shop/PaintShop.java index dbc5c04..2cfe561 100644 --- a/src/main/java/org/openaudible/desktop/swt/util/shop/PaintShop.java +++ b/src/main/java/org/openaudible/desktop/swt/util/shop/PaintShop.java @@ -4,7 +4,6 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.eclipse.swt.graphics.*; import org.eclipse.swt.widgets.Display; -import org.eclipse.swt.widgets.Shell; import org.openaudible.desktop.swt.gui.GUI; import java.io.InputStream; diff --git a/src/main/java/org/openaudible/util/HTTPGet.java b/src/main/java/org/openaudible/util/HTTPGet.java index 0ee7cf5..d8228b8 100644 --- a/src/main/java/org/openaudible/util/HTTPGet.java +++ b/src/main/java/org/openaudible/util/HTTPGet.java @@ -1,7 +1,5 @@ package org.openaudible.util; -import com.google.gson.JsonObject; -import com.google.gson.JsonParser; import org.apache.http.HttpEntity; import org.apache.http.HttpStatus; import org.apache.http.client.methods.CloseableHttpResponse; @@ -9,13 +7,14 @@ import org.apache.http.client.methods.HttpGet; import org.apache.http.impl.client.CloseableHttpClient; import org.apache.http.impl.client.HttpClients; import org.apache.http.util.EntityUtils; +import org.json.JSONObject; import java.io.IOException; public enum HTTPGet { instance; - public JsonObject getJSON(String url) throws IOException { + public JSONObject getJSON(String url) throws IOException { try (CloseableHttpClient httpclient = HttpClients.createDefault()) { HttpGet httpget = new HttpGet(url); try (CloseableHttpResponse httpResponse = httpclient.execute(httpget)) { @@ -23,9 +22,8 @@ public enum HTTPGet { HttpEntity httpEntity = httpResponse.getEntity(); if (httpResponse.getStatusLine().getStatusCode() == HttpStatus.SC_OK) { String entity = EntityUtils.toString(httpEntity); - JsonParser parser = new JsonParser(); - JsonObject obj = parser.parse(entity).getAsJsonObject(); - return obj; + return new JSONObject(entity); + } throw new IOException(httpResponse.getStatusLine().getReasonPhrase()); } diff --git a/src/main/java/org/openaudible/util/Util.java b/src/main/java/org/openaudible/util/Util.java index 0be25c1..c781cc9 100644 --- a/src/main/java/org/openaudible/util/Util.java +++ b/src/main/java/org/openaudible/util/Util.java @@ -1,7 +1,13 @@ package org.openaudible.util; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import java.util.HashMap; + public enum Util { instance; + private static final Log LOG = LogFactory.getLog(Util.class); public String timeString(long l, boolean includeHours) { // return ""+l; @@ -115,6 +121,35 @@ public enum Util { return s; } + + public static HashMap urlGetArgs(String url) { + HashMap map = new HashMap(); + + if (url.contains("?")) { + + try { + String args = url.substring(url.indexOf("?") + 1, url.length()); + + String split[] = args.split("&"); + for (String params : split) { + String kv[] = params.split("="); + if (kv.length == 2) { + map.put(kv[0], kv[1]); + + } else { + LOG.error("bad url param:" + params + " for " + url); /// happens when there is an & in title. + } + } + } catch (Throwable th) { + LOG.error("Error parsing url:" + url + " for args."); + } + } + return map; + + + } + + public static String cleanString(String out) { out = replaceAll(out, "\r", "\n"); out = replaceAll(out, " ", " ");