From a7ee2b74a6e79ffc3127a344c100c304ab4449c4 Mon Sep 17 00:00:00 2001 From: openaudible Date: Fri, 14 Sep 2018 03:10:45 -0700 Subject: [PATCH] testing --- .gitignore | 2 +- src/main/java/org/openaudible/Audible.java | 55 ++++++- .../openaudible/audible/AudibleScraper.java | 2 +- .../openaudible/audible/KindleScraper.java | 147 ------------------ src/main/java/org/openaudible/books/Book.java | 54 +++++-- .../desktop/swt/manager/AudibleGUI.java | 12 +- .../desktop/swt/manager/Version.java | 2 +- .../desktop/swt/manager/menu/Command.java | 21 ++- .../swt/manager/views/AudibleBrowser.java | 116 ++++++++++++-- .../swt/manager/views/AudibleWebPage.java | 120 ++++++++++++++ .../java/org/openaudible/util/HTMLUtil.java | 7 + 11 files changed, 348 insertions(+), 190 deletions(-) delete mode 100644 src/main/java/org/openaudible/audible/KindleScraper.java create mode 100644 src/main/java/org/openaudible/desktop/swt/manager/views/AudibleWebPage.java diff --git a/.gitignore b/.gitignore index b06e5ea..0faa210 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ install4j/ installers/ *.aax - +unused/ #generic tmp/ *.log diff --git a/src/main/java/org/openaudible/Audible.java b/src/main/java/org/openaudible/Audible.java index f84b076..c45ea58 100644 --- a/src/main/java/org/openaudible/Audible.java +++ b/src/main/java/org/openaudible/Audible.java @@ -322,6 +322,8 @@ public class Audible implements IQueueListener { for (Book b : bookList) { File aax = getAAXFileDest(b); aaxs.remove(aax); + + } if (aaxs.size() > 0) { @@ -334,6 +336,7 @@ public class Audible implements IQueueListener { try { Book b = AAXParser.instance.parseAAX(f, Directories.getDir(Directories.ART), AAXParser.CoverImageAction.saveInDirectory); takeBook(b); + task.setSubTask(b.toString()); File imageDest = Audible.instance.getImageFileDest(b); if (!imageDest.exists()) { @@ -359,6 +362,11 @@ public class Audible implements IQueueListener { for (Book b : bookList) { File mp3 = getMP3FileDest(b); mp3s.remove(mp3); + + + updatePurchaseDate(b); + + } if (mp3s.size() > 0) { @@ -388,6 +396,38 @@ public class Audible implements IQueueListener { } + private void updatePurchaseDate(Book b) { + if (b.getFullTitle().contains("A Game of T")) { + System.currentTimeMillis(); + } + String curDate = b.getPurchaseDate(); + boolean needUpdate = curDate.isEmpty(); + if (curDate.length() > 10) { + System.currentTimeMillis(); + b.setPurchaseDate(""); + needUpdate=true; + } + + if (needUpdate) { + File f = getAAXFileDest(b); + if (f.exists()) { + long lm = f.lastModified(); + if (lm == 0) lm = System.currentTimeMillis(); + + b.setPurchaseDate(new Date(lm)); + + String check = b.getPurchaseDateSortable(); + if (check.length()!=10) + { + b.setPurchaseDate(""); + } + + } + } + + + } + public void updateFileCache() { mp3Files = getFileSet(Directories.MP3); aaxFiles = getFileSet(Directories.AAX); @@ -409,10 +449,11 @@ public class Audible implements IQueueListener { } - public int getDownloadCount() { synchronized (lock) { - - return toDownload.size(); - } + public int getDownloadCount() { + synchronized (lock) { + + return toDownload.size(); + } } public int getConvertCount() { @@ -423,8 +464,10 @@ public class Audible implements IQueueListener { } - public int mp3Count() {synchronized (lock){ - return mp3Files.size();} + public int mp3Count() { + synchronized (lock) { + return mp3Files.size(); + } } diff --git a/src/main/java/org/openaudible/audible/AudibleScraper.java b/src/main/java/org/openaudible/audible/AudibleScraper.java index 323d948..87301ba 100644 --- a/src/main/java/org/openaudible/audible/AudibleScraper.java +++ b/src/main/java/org/openaudible/audible/AudibleScraper.java @@ -33,7 +33,7 @@ public class AudibleScraper { static int maxLoginAttempts = 1; final AudibleAccountPrefs account; private final AudibleClient webClient; - public HtmlPage page; + private HtmlPage page; boolean debugCust = false; boolean loggedIn = false; String clickToDownload = "Click to download "; diff --git a/src/main/java/org/openaudible/audible/KindleScraper.java b/src/main/java/org/openaudible/audible/KindleScraper.java deleted file mode 100644 index 25c0617..0000000 --- a/src/main/java/org/openaudible/audible/KindleScraper.java +++ /dev/null @@ -1,147 +0,0 @@ -package org.openaudible.audible; - -import com.gargoylesoftware.htmlunit.Page; -import com.gargoylesoftware.htmlunit.html.HtmlAnchor; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.openaudible.AudibleAccountPrefs; -import org.openaudible.Directories; -import org.openaudible.util.HTMLUtil; - -import java.io.File; -import java.io.IOException; -import java.util.ArrayList; -import java.util.HashSet; - -public class KindleScraper extends AudibleScraper { - public static KindleScraper instance; - private static final Log LOG = LogFactory.getLog(KindleScraper.class); - HashSet digitalProducts = new HashSet<>(); - int maxPages = 1; - - - public KindleScraper(AudibleAccountPrefs account) { - super(account); - instance = this; - } - - public void test() throws Exception { - home(); - if (!loggedIn()) - login(); - - getFreeBooks(); - } - - - public HtmlAnchor findBooks() throws IOException { - - HtmlAnchor next = null; - ArrayList nextLinks = new ArrayList<>(); - - for (HtmlAnchor n : page.getAnchors()) { - boolean print = false; - String dp = getDigitalProduct(n); - if (dp != null) { - digitalProducts.add(dp); - // print = true; - } - - if (n.toString().toLowerCase().contains("next")) { - print = true; - String clz = n.getAttribute("class"); - if (clz == null) clz = ""; - - if ("pagnNextLink".equalsIgnoreCase(n.getId()) || clz.contains("pagnNext"))// .equalsIgnoreCase(n.getAttribute("class"))) - next = n; - nextLinks.add(n); - } - if (print) { - LOG.info(n); - // LOG.info(n.getHrefAttribute()); - } - } - - - LOG.info("next links:" + nextLinks.size()); - LOG.info("digitalProducts:" + digitalProducts.size()); - return next; - } - - private String getDigitalProduct(HtmlAnchor n) { - String find = "/dp/"; - String ref = n.getHrefAttribute(); - int ch = ref.indexOf(find); - - if (ch != -1) { - String id = ref.substring(ch + find.length(), ref.length()); - ch = id.indexOf("/"); - assert (ch != -1); - if (ch != -1) { - id = id.substring(0, ch); - int len = id.length(); - assert (len < 15); - if (len < 15) - return id; - } - - - } - return null; - } - - public void getProductInfo(String dp) throws IOException { - - File x = new File(Directories.getTmpDir(), "dp_" + dp + ".html"); - if (!x.exists()) { - String base = "https://www.amazon.com/dp/" + dp; - setURL(base); - parseDigitalProductPage(page); - HTMLUtil.writeFile(x, page.getDocumentElement().asXml()); - } - - - } - - private void parseDigitalProductPage(Page p) { - - - } - - public String getAudibleBase() { - return "https://amazon.com/"; - - // return account.audibleRegion.getBaseURL(); - } - - - public void getFreeBooks() throws IOException { - int pages = 0; - - - String base = "https://www.amazon.com/Books-with-Narration-in-Kindle-Unlimited/b?ie=UTF8&node=9630682011"; - - setURL(base); - - checkLoggedIn(); - - for (; ; ) { - int startBooks = digitalProducts.size(); - HtmlAnchor next = findBooks(); - if (next == null) break; - - int endBooks = digitalProducts.size(); - if (startBooks == endBooks) break; - if (++pages > maxPages) break; - - page = next.click(); - } - LOG.info("digitalProducts:" + digitalProducts.size()); - for (String dp : digitalProducts) { - getProductInfo(dp); - } - - } - -} - diff --git a/src/main/java/org/openaudible/books/Book.java b/src/main/java/org/openaudible/books/Book.java index caecbf5..a7c7bb5 100644 --- a/src/main/java/org/openaudible/books/Book.java +++ b/src/main/java/org/openaudible/books/Book.java @@ -10,6 +10,9 @@ import java.util.Date; import java.util.HashMap; public class Book implements Comparable, Serializable { + static SimpleDateFormat audibleDateFormat = new SimpleDateFormat("dd-MMM-yyyy"); + static SimpleDateFormat purchaseDateFormat = new SimpleDateFormat("MM-dd-yyyy"); + static SimpleDateFormat dispalyFormat = new SimpleDateFormat("yyyy-MM-dd"); private final HashMap map = new HashMap<>(); public Book(HashMap m) { @@ -213,26 +216,26 @@ public class Book implements Comparable, Serializable { return get(BookElement.rating_average); } - public void setRating_average(double rating_average) { - set(BookElement.rating_average, "" + rating_average); - } - public void setRating_average(String rating_average) { set(BookElement.rating_average, rating_average); } + public void setRating_average(double rating_average) { + set(BookElement.rating_average, "" + rating_average); + } + public String getRating_count() { return get(BookElement.rating_count); } - public void setRating_count(int rating_count) { - set(BookElement.rating_count, "" + rating_count); - } - public void setRating_count(String rating_count) { set(BookElement.rating_count, rating_count); } + public void setRating_count(int rating_count) { + set(BookElement.rating_count, "" + rating_count); + } + public String getRelease_date() { return get(BookElement.release_date); } @@ -288,6 +291,11 @@ public class Book implements Comparable, Serializable { return get(BookElement.purchase_date); } + public void setPurchaseDate(Date d) { + String s = dispalyFormat.format(d); + setPurchaseDate(s); + } + public void setPurchaseDate(String purchaseDateText) { set(BookElement.purchase_date, purchaseDateText); } @@ -301,10 +309,9 @@ public class Book implements Comparable, Serializable { String date = getRelease_date(); if (!date.isEmpty()) { // 11-MAR-2015 - SimpleDateFormat parseFormat = new SimpleDateFormat("dd-MMM-yyyy"); + try { - Date d = parseFormat.parse(date); - SimpleDateFormat dispalyFormat = new SimpleDateFormat("yyyy-MM-dd"); + Date d = audibleDateFormat.parse(date); String out = dispalyFormat.format(d); return out; } catch (ParseException e) { @@ -318,14 +325,33 @@ public class Book implements Comparable, Serializable { public String getPurchaseDateSortable() { String date = getPurchaseDate(); + // if (true) return date; + if (!date.isEmpty()) { + + boolean err = false; + + String dt[] = date.split("-"); if (dt.length == 3) { - return "20" + dt[2] + "-" + dt[0] + "-" + dt[1]; // yyyy-mm-dd for sorting and viewing - // warning, y3k bug - } else { + if (dt[0].length()==4&&dt[1].length()==2&&dt[2].length()==2) + { + return date; // yyyy-mm-dd format we like + } + if (dt[0].length()==2&&dt[1].length()==2&&dt[2].length()==2) + { + return "20" + dt[2] + "-" + dt[0] + "-" + dt[1]; // mm-dd-yy is most common + } + err = true; + } else { + err = true; } + + if (err) { + return "* " + date; + } + } return date; diff --git a/src/main/java/org/openaudible/desktop/swt/manager/AudibleGUI.java b/src/main/java/org/openaudible/desktop/swt/manager/AudibleGUI.java index a2946c9..dad067e 100644 --- a/src/main/java/org/openaudible/desktop/swt/manager/AudibleGUI.java +++ b/src/main/java/org/openaudible/desktop/swt/manager/AudibleGUI.java @@ -851,16 +851,14 @@ public class AudibleGUI implements BookListener, ConnectionListener { } public void test1() { - if (KindleScraper.instance == null) { - new KindleScraper(audible.getAccount()); - - } + if (browser==null) + browse("http://audible.com/lib"); + try { - KindleScraper.instance.test(); - + browser.test(); } catch (Throwable th) { - LOG.debug("Error", th); + LOG.info("Error", th); } } diff --git a/src/main/java/org/openaudible/desktop/swt/manager/Version.java b/src/main/java/org/openaudible/desktop/swt/manager/Version.java index 68c9e1a..ef83a6b 100644 --- a/src/main/java/org/openaudible/desktop/swt/manager/Version.java +++ b/src/main/java/org/openaudible/desktop/swt/manager/Version.java @@ -4,7 +4,7 @@ public interface Version { String appName = "OpenAudible"; String appVersion = "1.1.8"; - boolean appDebug = false; + boolean appDebug = true; String appLink = "http://openaudible.org"; String versionLink = "http://openaudible.org/swt_version.json"; String news = "test news here\nok!"; // latest features go here. diff --git a/src/main/java/org/openaudible/desktop/swt/manager/menu/Command.java b/src/main/java/org/openaudible/desktop/swt/manager/menu/Command.java index 4ea9e45..6ce4a77 100644 --- a/src/main/java/org/openaudible/desktop/swt/manager/menu/Command.java +++ b/src/main/java/org/openaudible/desktop/swt/manager/menu/Command.java @@ -57,8 +57,10 @@ public enum Command { case About: break; case ViewInAudible: + cmd='I'; break; case Show_MP3: + cmd='M'; break; case Show_AAX: break; @@ -68,6 +70,8 @@ public enum Command { break; case Export_Book_List: break; + case Import_AAX_Files: + break; case Refresh_Book_Info: break; case Connect: @@ -87,11 +91,26 @@ public enum Command { break; case Convert_All: break; + case Help: + cmd='H'; + break; case Browser: cmd = 'B'; break; + case Check_For_Update: + cmd='U'; + break; + case AppWebPage: + break; case Logout_and_Clear_Cookies: - case Console: + case Console:cmd = 'L'; + break; + case Test1: + cmd='T'; + break; + case MenuSeparator: + break; + case Ignore_Book: break; } return cmd; diff --git a/src/main/java/org/openaudible/desktop/swt/manager/views/AudibleBrowser.java b/src/main/java/org/openaudible/desktop/swt/manager/views/AudibleBrowser.java index 8966816..754c9a1 100644 --- a/src/main/java/org/openaudible/desktop/swt/manager/views/AudibleBrowser.java +++ b/src/main/java/org/openaudible/desktop/swt/manager/views/AudibleBrowser.java @@ -18,14 +18,17 @@ import org.openaudible.audible.AudibleClient; import org.openaudible.audible.ConnectionNotifier; import org.openaudible.desktop.swt.gui.MessageBoxFactory; import org.openaudible.desktop.swt.gui.SWTAsync; +import org.openaudible.util.HTMLUtil; import org.openaudible.util.Platform; import java.io.File; +import java.io.IOException; import java.net.URI; import java.net.URL; import java.util.ArrayList; import java.util.Collection; + public class AudibleBrowser { public final static Log logger = LogFactory.getLog(AudibleBrowser.class); int index; @@ -56,18 +59,26 @@ public class AudibleBrowser { browser = new Browser(parent, style); browser.addTitleListener(event -> getShell().setText(event.title)); - - browser.addProgressListener( new ProgressAdapter() { + + browser.addProgressListener(new ProgressAdapter() { @Override - public void completed( ProgressEvent event ) { + public void completed(ProgressEvent event) { String text = browser.getText(); ConnectionNotifier.instance.pageLoaded(browser.getUrl(), browser.getText()); - + // System.out.println("**** HTML **** \n"+text); + + try { + File f = HTMLUtil.debugToFile("last.html", text); + System.out.println("Loaded page:" + f.getAbsolutePath()); + } catch (IOException e) { + e.printStackTrace(); + } + // here, text will contain the full page source } - } ); - - + }); + + Object t = browser.getWebBrowser(); customHeader = new String[1]; @@ -418,8 +429,91 @@ public class AudibleBrowser { shell.open(); } - private void test() { + String get(String w) { + if (!w.endsWith(";")) + w += ";"; + + return eval("return " + w); + } + String eval(String w) { + try { + Object o = browser.evaluate(w); + String clz = o != null ? o.getClass().toString() : ""; + System.out.println("eval(" + w + ") -> " + o + " " + clz); + + if (o != null) { + if (o instanceof String) + return (String) o; + return o.toString(); + } + + } catch (Throwable th) { + logger.error("unable to eval " + w, th); + } + return null; + } + + public void nextPage() + { + String js = "const buttons = document.querySelectorAll('button');\n" + + "var next=null;\n" + + "var test='len='+buttons.length;\n" + + "\n" + + "for (var i = 0; i < buttons.length; i++) {\n" + + " var b = buttons[i];\n" + + "\ttest += 'b='+b;\n" + + "\tconst dn = b.getAttribute('data-name');\n" + + "\tif (dn!=null) { next = b; test = dn; }\t\n" + + "}\n" + + "\n" + + "\n" + + "if (next!=null) next.click();\n" + + "return test;\n" + + "\n" + + "\n"; + eval(js); + } + + public void test() { + + try { + if (true) + { + nextPage(); + return; + } + + Object window = eval("return window;"); + String userAgent = eval("return navigator.userAgent"); + String navigatorVendor = eval("return navigator.vendor"); + String numAnchors = eval("return document.anchors.length;"); + + eval("return document.anchors;"); + eval("return document;"); + get("document.getElementsByTagName('button')"); + get("document.getElementsByTagName('button')[0]"); + get("document.querySelectorAll('img')"); + get("document.querySelectorAll('img').length"); + String c1 = "const buttons = document.querySelectorAll('button');\n" + + " var test='';\n" + + "for (var i = 0; i < buttons.length; i++) {\n" + + " test += 'abc';\n" + + "}\nreturn test;"; + + eval(c1); + + +// for(var i = 0; i < inputs.length; i++) { +// if(inputs[i].type.toLowerCase() == 'text') { +// alert(inputs[i].value); +// } +// } + + + } catch (Throwable th) { + th.printStackTrace(); + } } public void setUrl(String u) { @@ -482,8 +576,6 @@ public class AudibleBrowser { if (!isDisposed()) browser.close(); browser = null; } - - - - + + } diff --git a/src/main/java/org/openaudible/desktop/swt/manager/views/AudibleWebPage.java b/src/main/java/org/openaudible/desktop/swt/manager/views/AudibleWebPage.java new file mode 100644 index 0000000..2e66487 --- /dev/null +++ b/src/main/java/org/openaudible/desktop/swt/manager/views/AudibleWebPage.java @@ -0,0 +1,120 @@ +package org.openaudible.desktop.swt.manager.views; + +import com.gargoylesoftware.htmlunit.History; +import com.gargoylesoftware.htmlunit.Page; +import com.gargoylesoftware.htmlunit.WebClient; +import com.gargoylesoftware.htmlunit.WebWindow; +import com.gargoylesoftware.htmlunit.javascript.background.JavaScriptJobManager; +import net.sourceforge.htmlunit.corejs.javascript.ScriptableObject; +import org.eclipse.swt.widgets.Composite; + + +public class AudibleWebPage extends AudibleBrowser implements WebWindow { + + String name = ""; + + public AudibleWebPage(Composite parent, String url) { + super(parent, url); + } + + @Override + public String getName() { + return name; + } + + @Override + public void setName(String s) { + name = s; + } + + @Override + public Page getEnclosedPage() { + return null; + } + + @Override + public void setEnclosedPage(Page page) { + + } + + @Override + public WebWindow getParentWindow() { + return null; + } + + @Override + public WebWindow getTopWindow() { + return this; + } + + @Override + public WebClient getWebClient() { + assert (false); + return null; + } + + @Override + public History getHistory() { + return null; + } + + @Override + public void setScriptableObject(ScriptableObject scriptableObject) { + + } + + @Override + public ScriptableObject getScriptableObject() { + return null; + } + + @Override + public JavaScriptJobManager getJobManager() { + return null; + } + + @Override + public boolean isClosed() { + return false; + } + + @Override + public int getInnerWidth() { + return 0; + } + + @Override + public void setInnerWidth(int i) { + + } + + @Override + public int getOuterWidth() { + return 0; + } + + @Override + public void setOuterWidth(int i) { + + } + + @Override + public int getInnerHeight() { + return 0; + } + + @Override + public void setInnerHeight(int i) { + + } + + @Override + public int getOuterHeight() { + return 0; + } + + @Override + public void setOuterHeight(int i) { + + } +} diff --git a/src/main/java/org/openaudible/util/HTMLUtil.java b/src/main/java/org/openaudible/util/HTMLUtil.java index a32348d..be767ea 100644 --- a/src/main/java/org/openaudible/util/HTMLUtil.java +++ b/src/main/java/org/openaudible/util/HTMLUtil.java @@ -1,6 +1,7 @@ package org.openaudible.util; import com.gargoylesoftware.htmlunit.html.*; +import org.apache.commons.io.FileUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.openaudible.Directories; @@ -94,6 +95,12 @@ public class HTMLUtil { return new File(Directories.getTmpDir(), what); } + public static File debugToFile(String what,String text) throws IOException { + File f = debugFile(what); + FileUtils.writeByteArrayToFile(f, text.getBytes()); + return f; + } + public static String debugNode(DomNode p, String what) { String xml = "";