wbrawner/openaudible
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

testing

Browse source
This commit is contained in:
openaudible 2018-09-14 03:10:45 -07:00
parent b6e81920c6
commit a7ee2b74a6
11 changed files with 348 additions and 190 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -1,7 +1,7 @@
install4j/
installers/
*.aax
unused/
#generic
tmp/
*.log

55
src/main/java/org/openaudible/Audible.java
View file

@ -322,6 +322,8 @@ public class Audible implements IQueueListener<Book> {
for (Book b : bookList) {
File aax = getAAXFileDest(b);
aaxs.remove(aax);
}
if (aaxs.size() > 0) {
@ -334,6 +336,7 @@ public class Audible implements IQueueListener<Book> {
try {
Book b = AAXParser.instance.parseAAX(f, Directories.getDir(Directories.ART), AAXParser.CoverImageAction.saveInDirectory);
takeBook(b);
task.setSubTask(b.toString());
File imageDest = Audible.instance.getImageFileDest(b);
if (!imageDest.exists()) {
@ -359,6 +362,11 @@ public class Audible implements IQueueListener<Book> {
for (Book b : bookList) {
File mp3 = getMP3FileDest(b);
mp3s.remove(mp3);
updatePurchaseDate(b);
}
if (mp3s.size() > 0) {
@ -388,6 +396,38 @@ public class Audible implements IQueueListener<Book> {
}
private void updatePurchaseDate(Book b) {
if (b.getFullTitle().contains("A Game of T")) {
System.currentTimeMillis();
}
String curDate = b.getPurchaseDate();
boolean needUpdate = curDate.isEmpty();
if (curDate.length() > 10) {
System.currentTimeMillis();
b.setPurchaseDate("");
needUpdate=true;
}
if (needUpdate) {
File f = getAAXFileDest(b);
if (f.exists()) {
long lm = f.lastModified();
if (lm == 0) lm = System.currentTimeMillis();
b.setPurchaseDate(new Date(lm));
String check = b.getPurchaseDateSortable();
if (check.length()!=10)
{
b.setPurchaseDate("");
}
}
}
}
public void updateFileCache() {
mp3Files = getFileSet(Directories.MP3);
aaxFiles = getFileSet(Directories.AAX);
@ -409,10 +449,11 @@ public class Audible implements IQueueListener<Book> {
}
public int getDownloadCount() { synchronized (lock) {
return toDownload.size();
}
public int getDownloadCount() {
synchronized (lock) {
return toDownload.size();
}
}
public int getConvertCount() {
@ -423,8 +464,10 @@ public class Audible implements IQueueListener<Book> {
}
public int mp3Count() {synchronized (lock){
return mp3Files.size();}
public int mp3Count() {
synchronized (lock) {
return mp3Files.size();
}
}

2
src/main/java/org/openaudible/audible/AudibleScraper.java
View file

@ -33,7 +33,7 @@ public class AudibleScraper {
static int maxLoginAttempts = 1;
final AudibleAccountPrefs account;
private final AudibleClient webClient;
public HtmlPage page;
private HtmlPage page;
boolean debugCust = false;
boolean loggedIn = false;
String clickToDownload = "Click to download ";

147
src/main/java/org/openaudible/audible/KindleScraper.java
View file

@ -1,147 +0,0 @@
package org.openaudible.audible;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.openaudible.AudibleAccountPrefs;
import org.openaudible.Directories;
import org.openaudible.util.HTMLUtil;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
public class KindleScraper extends AudibleScraper {
public static KindleScraper instance;
private static final Log LOG = LogFactory.getLog(KindleScraper.class);
HashSet<String> digitalProducts = new HashSet<>();
int maxPages = 1;
public KindleScraper(AudibleAccountPrefs account) {
super(account);
instance = this;
}
public void test() throws Exception {
home();
if (!loggedIn())
login();
getFreeBooks();
}
public HtmlAnchor findBooks() throws IOException {
HtmlAnchor next = null;
ArrayList<HtmlAnchor> nextLinks = new ArrayList<>();
for (HtmlAnchor n : page.getAnchors()) {
boolean print = false;
String dp = getDigitalProduct(n);
if (dp != null) {
digitalProducts.add(dp);
// print = true;
}
if (n.toString().toLowerCase().contains("next")) {
print = true;
String clz = n.getAttribute("class");
if (clz == null) clz = "";
if ("pagnNextLink".equalsIgnoreCase(n.getId()) || clz.contains("pagnNext"))// .equalsIgnoreCase(n.getAttribute("class")))
next = n;
nextLinks.add(n);
}
if (print) {
LOG.info(n);
// LOG.info(n.getHrefAttribute());
}
}
LOG.info("next links:" + nextLinks.size());
LOG.info("digitalProducts:" + digitalProducts.size());
return next;
}
private String getDigitalProduct(HtmlAnchor n) {
String find = "/dp/";
String ref = n.getHrefAttribute();
int ch = ref.indexOf(find);
if (ch != -1) {
String id = ref.substring(ch + find.length(), ref.length());
ch = id.indexOf("/");
assert (ch != -1);
if (ch != -1) {
id = id.substring(0, ch);
int len = id.length();
assert (len < 15);
if (len < 15)
return id;
}
}
return null;
}
public void getProductInfo(String dp) throws IOException {
File x = new File(Directories.getTmpDir(), "dp_" + dp + ".html");
if (!x.exists()) {
String base = "https://www.amazon.com/dp/" + dp;
setURL(base);
parseDigitalProductPage(page);
HTMLUtil.writeFile(x, page.getDocumentElement().asXml());
}
}
private void parseDigitalProductPage(Page p) {
}
public String getAudibleBase() {
return "https://amazon.com/";
// return account.audibleRegion.getBaseURL();
}
public void getFreeBooks() throws IOException {
int pages = 0;
String base = "https://www.amazon.com/Books-with-Narration-in-Kindle-Unlimited/b?ie=UTF8&node=9630682011";
setURL(base);
checkLoggedIn();
for (; ; ) {
int startBooks = digitalProducts.size();
HtmlAnchor next = findBooks();
if (next == null) break;
int endBooks = digitalProducts.size();
if (startBooks == endBooks) break;
if (++pages > maxPages) break;
page = next.click();
}
LOG.info("digitalProducts:" + digitalProducts.size());
for (String dp : digitalProducts) {
getProductInfo(dp);
}
}
}

54
src/main/java/org/openaudible/books/Book.java
View file

@ -10,6 +10,9 @@ import java.util.Date;
import java.util.HashMap;
public class Book implements Comparable<Book>, Serializable {
static SimpleDateFormat audibleDateFormat = new SimpleDateFormat("dd-MMM-yyyy");
static SimpleDateFormat purchaseDateFormat = new SimpleDateFormat("MM-dd-yyyy");
static SimpleDateFormat dispalyFormat = new SimpleDateFormat("yyyy-MM-dd");
private final HashMap<String, String> map = new HashMap<>();
public Book(HashMap<String, String> m) {
@ -213,26 +216,26 @@ public class Book implements Comparable<Book>, Serializable {
return get(BookElement.rating_average);
}
public void setRating_average(double rating_average) {
set(BookElement.rating_average, "" + rating_average);
}
public void setRating_average(String rating_average) {
set(BookElement.rating_average, rating_average);
}
public void setRating_average(double rating_average) {
set(BookElement.rating_average, "" + rating_average);
}
public String getRating_count() {
return get(BookElement.rating_count);
}
public void setRating_count(int rating_count) {
set(BookElement.rating_count, "" + rating_count);
}
public void setRating_count(String rating_count) {
set(BookElement.rating_count, rating_count);
}
public void setRating_count(int rating_count) {
set(BookElement.rating_count, "" + rating_count);
}
public String getRelease_date() {
return get(BookElement.release_date);
}
@ -288,6 +291,11 @@ public class Book implements Comparable<Book>, Serializable {
return get(BookElement.purchase_date);
}
public void setPurchaseDate(Date d) {
String s = dispalyFormat.format(d);
setPurchaseDate(s);
}
public void setPurchaseDate(String purchaseDateText) {
set(BookElement.purchase_date, purchaseDateText);
}
@ -301,10 +309,9 @@ public class Book implements Comparable<Book>, Serializable {
String date = getRelease_date();
if (!date.isEmpty()) {
// 11-MAR-2015
SimpleDateFormat parseFormat = new SimpleDateFormat("dd-MMM-yyyy");
try {
Date d = parseFormat.parse(date);
SimpleDateFormat dispalyFormat = new SimpleDateFormat("yyyy-MM-dd");
Date d = audibleDateFormat.parse(date);
String out = dispalyFormat.format(d);
return out;
} catch (ParseException e) {
@ -318,14 +325,33 @@ public class Book implements Comparable<Book>, Serializable {
public String getPurchaseDateSortable() {
String date = getPurchaseDate();
// if (true) return date;
if (!date.isEmpty()) {
boolean err = false;
String dt[] = date.split("-");
if (dt.length == 3) {
return "20" + dt[2] + "-" + dt[0] + "-" + dt[1]; // yyyy-mm-dd for sorting and viewing
// warning, y3k bug
} else {
if (dt[0].length()==4&&dt[1].length()==2&&dt[2].length()==2)
{
return date; // yyyy-mm-dd format we like
}
if (dt[0].length()==2&&dt[1].length()==2&&dt[2].length()==2)
{
return "20" + dt[2] + "-" + dt[0] + "-" + dt[1]; // mm-dd-yy is most common
}
err = true;
} else {
err = true;
}
if (err) {
return "* " + date;
}
}
return date;

12
src/main/java/org/openaudible/desktop/swt/manager/AudibleGUI.java
View file

@ -851,16 +851,14 @@ public class AudibleGUI implements BookListener, ConnectionListener {
}
public void test1() {
if (KindleScraper.instance == null) {
new KindleScraper(audible.getAccount());
}
if (browser==null)
browse("http://audible.com/lib");
try {
KindleScraper.instance.test();
browser.test();
} catch (Throwable th) {
LOG.debug("Error", th);
LOG.info("Error", th);
}
}

2
src/main/java/org/openaudible/desktop/swt/manager/Version.java
View file

@ -4,7 +4,7 @@ public interface Version {
String appName = "OpenAudible";
String appVersion = "1.1.8";
boolean appDebug = false;
boolean appDebug = true;
String appLink = "http://openaudible.org";
String versionLink = "http://openaudible.org/swt_version.json";
String news = "test news here\nok!"; // latest features go here.

21
src/main/java/org/openaudible/desktop/swt/manager/menu/Command.java
View file

@ -57,8 +57,10 @@ public enum Command {
case About:
break;
case ViewInAudible:
cmd='I';
break;
case Show_MP3:
cmd='M';
break;
case Show_AAX:
break;
@ -68,6 +70,8 @@ public enum Command {
break;
case Export_Book_List:
break;
case Import_AAX_Files:
break;
case Refresh_Book_Info:
break;
case Connect:
@ -87,11 +91,26 @@ public enum Command {
break;
case Convert_All:
break;
case Help:
cmd='H';
break;
case Browser:
cmd = 'B';
break;
case Check_For_Update:
cmd='U';
break;
case AppWebPage:
break;
case Logout_and_Clear_Cookies:
case Console:
case Console:cmd = 'L';
break;
case Test1:
cmd='T';
break;
case MenuSeparator:
break;
case Ignore_Book:
break;
}
return cmd;

116
src/main/java/org/openaudible/desktop/swt/manager/views/AudibleBrowser.java
View file

@ -18,14 +18,17 @@ import org.openaudible.audible.AudibleClient;
import org.openaudible.audible.ConnectionNotifier;
import org.openaudible.desktop.swt.gui.MessageBoxFactory;
import org.openaudible.desktop.swt.gui.SWTAsync;
import org.openaudible.util.HTMLUtil;
import org.openaudible.util.Platform;
import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URL;
import java.util.ArrayList;
import java.util.Collection;
public class AudibleBrowser {
public final static Log logger = LogFactory.getLog(AudibleBrowser.class);
int index;
@ -56,18 +59,26 @@ public class AudibleBrowser {
browser = new Browser(parent, style);
browser.addTitleListener(event -> getShell().setText(event.title));
browser.addProgressListener( new ProgressAdapter() {
browser.addProgressListener(new ProgressAdapter() {
@Override
public void completed( ProgressEvent event ) {
public void completed(ProgressEvent event) {
String text = browser.getText();
ConnectionNotifier.instance.pageLoaded(browser.getUrl(), browser.getText());
// System.out.println("**** HTML **** \n"+text);
try {
File f = HTMLUtil.debugToFile("last.html", text);
System.out.println("Loaded page:" + f.getAbsolutePath());
} catch (IOException e) {
e.printStackTrace();
}
// here, text will contain the full page source
}
} );
});
Object t = browser.getWebBrowser();
customHeader = new String[1];
@ -418,8 +429,91 @@ public class AudibleBrowser {
shell.open();
}
private void test() {
String get(String w) {
if (!w.endsWith(";"))
w += ";";
return eval("return " + w);
}
String eval(String w) {
try {
Object o = browser.evaluate(w);
String clz = o != null ? o.getClass().toString() : "";
System.out.println("eval(" + w + ") -> " + o + " " + clz);
if (o != null) {
if (o instanceof String)
return (String) o;
return o.toString();
}
} catch (Throwable th) {
logger.error("unable to eval " + w, th);
}
return null;
}
public void nextPage()
{
String js = "const buttons = document.querySelectorAll('button');\n" +
"var next=null;\n" +
"var test='len='+buttons.length;\n" +
"\n" +
"for (var i = 0; i < buttons.length; i++) {\n" +
" var b = buttons[i];\n" +
"\ttest += 'b='+b;\n" +
"\tconst dn = b.getAttribute('data-name');\n" +
"\tif (dn!=null) { next = b; test = dn; }\t\n" +
"}\n" +
"\n" +
"\n" +
"if (next!=null) next.click();\n" +
"return test;\n" +
"\n" +
"\n";
eval(js);
}
public void test() {
try {
if (true)
{
nextPage();
return;
}
Object window = eval("return window;");
String userAgent = eval("return navigator.userAgent");
String navigatorVendor = eval("return navigator.vendor");
String numAnchors = eval("return document.anchors.length;");
eval("return document.anchors;");
eval("return document;");
get("document.getElementsByTagName('button')");
get("document.getElementsByTagName('button')[0]");
get("document.querySelectorAll('img')");
get("document.querySelectorAll('img').length");
String c1 = "const buttons = document.querySelectorAll('button');\n"
+ " var test='';\n"
+ "for (var i = 0; i < buttons.length; i++) {\n"
+ " test += 'abc';\n"
+ "}\nreturn test;";
eval(c1);
// for(var i = 0; i < inputs.length; i++) {
// if(inputs[i].type.toLowerCase() == 'text') {
// alert(inputs[i].value);
// }
// }
} catch (Throwable th) {
th.printStackTrace();
}
}
public void setUrl(String u) {
@ -482,8 +576,6 @@ public class AudibleBrowser {
if (!isDisposed()) browser.close();
browser = null;
}
}

120
src/main/java/org/openaudible/desktop/swt/manager/views/AudibleWebPage.java Normal file
View file

@ -0,0 +1,120 @@
package org.openaudible.desktop.swt.manager.views;
import com.gargoylesoftware.htmlunit.History;
import com.gargoylesoftware.htmlunit.Page;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.WebWindow;
import com.gargoylesoftware.htmlunit.javascript.background.JavaScriptJobManager;
import net.sourceforge.htmlunit.corejs.javascript.ScriptableObject;
import org.eclipse.swt.widgets.Composite;
public class AudibleWebPage extends AudibleBrowser implements WebWindow {
String name = "";
public AudibleWebPage(Composite parent, String url) {
super(parent, url);
}
@Override
public String getName() {
return name;
}
@Override
public void setName(String s) {
name = s;
}
@Override
public Page getEnclosedPage() {
return null;
}
@Override
public void setEnclosedPage(Page page) {
}
@Override
public WebWindow getParentWindow() {
return null;
}
@Override
public WebWindow getTopWindow() {
return this;
}
@Override
public WebClient getWebClient() {
assert (false);
return null;
}
@Override
public History getHistory() {
return null;
}
@Override
public void setScriptableObject(ScriptableObject scriptableObject) {
}
@Override
public ScriptableObject getScriptableObject() {
return null;
}
@Override
public JavaScriptJobManager getJobManager() {
return null;
}
@Override
public boolean isClosed() {
return false;
}
@Override
public int getInnerWidth() {
return 0;
}
@Override
public void setInnerWidth(int i) {
}
@Override
public int getOuterWidth() {
return 0;
}
@Override
public void setOuterWidth(int i) {
}
@Override
public int getInnerHeight() {
return 0;
}
@Override
public void setInnerHeight(int i) {
}
@Override
public int getOuterHeight() {
return 0;
}
@Override
public void setOuterHeight(int i) {
}
}

7
src/main/java/org/openaudible/util/HTMLUtil.java
View file

@ -1,6 +1,7 @@
package org.openaudible.util;
import com.gargoylesoftware.htmlunit.html.*;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.openaudible.Directories;
@ -94,6 +95,12 @@ public class HTMLUtil {
return new File(Directories.getTmpDir(), what);
}
public static File debugToFile(String what,String text) throws IOException {
File f = debugFile(what);
FileUtils.writeByteArrayToFile(f, text.getBytes());
return f;
}
public static String debugNode(DomNode p, String what) {
String xml = "";