Merge pull request #61 from Choco31415/development

General Fixes

Author: Choco31415

src/main/java/WikiBot/ContentRep/PageLocation.java

@@ -54,11 +54,19 @@ public boolean equals(Object obj) {
 		}
 		if (obj.getClass().equals(PageLocation.class)) {
 			PageLocation pg = (PageLocation)obj;
-			return pg.getTitle().equalsIgnoreCase(titleObject.getTitle()) && lan.equals(pg.getLanguage());
+			boolean equal = true;
+			equal = equal & lan.equals(pg.getLanguage());
+			equal = equal & pg.getTitleObject().equals(titleObject);
+			return equal;
 		}
 		return false;
 	}
 
+	@Override
+    public int hashCode() {
+		return lan.hashCode() + getTitle().hashCode();
+	}
+	
 	@Override
 	public String toString() {
 		return "PageLocation: " + lan + ": " + titleObject.getTitle();

src/main/java/WikiBot/ContentRep/PageTitle.java

@@ -28,6 +28,23 @@ public PageTitle(String title_) {
 	public String getNameSpace() { return nameSpace; }
 	public String getTitleWithoutNameSpace() { return titleWithoutNameSpace; }
 
+	@Override
+	public boolean equals(Object obj) {
+		if (obj == null) {
+			return false;
+		}
+		if (obj.getClass().equals(PageLocation.class)) {
+			PageTitle pt = (PageTitle)obj;
+			return getTitle().substring(0).equalsIgnoreCase(pt.getTitle().substring(0));
+		}
+		return false;
+	}
+	
+	@Override
+    public int hashCode() {
+		return title.hashCode();
+	}
+	
 	@Override
 	public String toString() {
 		return "\nPageTitle: " + title;

src/main/java/WikiBot/Core/GenericBot.java

@@ -164,7 +164,7 @@ public SimplePage getWikiSimplePage(PageLocation loc) {
 	public boolean doesPageExist(PageLocation loc) {
 		JsonNode serverOutput = getWikiPageJsonCode(loc);
 
-		return serverOutput.findValue("missing") == null; // If contains missing tag, the page is missing.
+		return serverOutput.findValue("missing") == null && serverOutput.findValue("invalid") == null; // If contains missing tag, the page is missing.
 	}
 
 	private JsonNode getWikiPageJsonCode(PageLocation loc) {		
@@ -351,7 +351,8 @@ private void getPageRevisions(Page page) {
 	public ArrayList<Revision> getPastRevisions(PageLocation loc, int localRevisionDepth, boolean getContent) {
 		ArrayList<Revision> toReturn = new ArrayList<>();
 
-		String serverOutput = APIcommand(new QueryPageRevisions(loc, Math.min(localRevisionDepth, APIlimit), getContent));
+		int revDepth = Math.min(localRevisionDepth, APIlimit);
+		String serverOutput = APIcommand(new QueryPageRevisions(loc, revDepth, getContent));
 
 
 		// Read in the Json!!!
@@ -416,7 +417,8 @@ public ArrayList<Revision> getRecentChanges(String language, int depth) {
 
 		int revisionsNeeded = depth;
 		do {
-			int batchSize = Math.min(Math.min(30, APIlimit), revisionsNeeded);
+			revisionsNeeded = depth - toReturn.size();
+			int batchSize = Math.min(APIlimit, revisionsNeeded);
 
 			//Make a query call.
 			String serverOutput;
@@ -465,8 +467,6 @@ public ArrayList<Revision> getRecentChanges(String language, int depth) {
 				toReturn.add(revision);
 			}
 
-			revisionsNeeded -= batchSize;
-			
 			//Try continuing the query.
 			if (rootNode.findValue("rccontinue") != null) {
 				rccontinue = rootNode.findValue("rccontinue").asText();
@@ -475,7 +475,7 @@ public ArrayList<Revision> getRecentChanges(String language, int depth) {
 			} else {
 				rccontinue = null;
 			}
-		} while (rccontinue != null && revisionsNeeded != 0);
+		} while (rccontinue != null && toReturn.size() < depth);
 
 		 return toReturn;
 	}
@@ -618,9 +618,10 @@ public ArrayList<PageLocation> getPagesThatLinkTo(PageLocation loc, int depth) {
 
 		logFine("Getting pages that link to: "  + loc.getTitle());
 
-		int backlinksNeeded = depth;
+		int backlinksNeeded;
 		do {
 			//Make a query call.
+			backlinksNeeded = depth - toReturn.size();
 			int batchSize = Math.min(Math.min(30, APIlimit), backlinksNeeded);
 
 			String serverOutput;
@@ -651,8 +652,6 @@ public ArrayList<PageLocation> getPagesThatLinkTo(PageLocation loc, int depth) {
 				toReturn.add(backlinkLoc);
 			}
 
-			backlinksNeeded -= batchSize;
-			
 			//Try continuing the query.
 			if (rootNode.findValue("blcontinue") != null) {
 				blcontinue = rootNode.findValue("blcontinue").asText();
@@ -661,7 +660,7 @@ public ArrayList<PageLocation> getPagesThatLinkTo(PageLocation loc, int depth) {
 			} else {
 				blcontinue = null;
 			}
-		} while (blcontinue != null && backlinksNeeded != 0);
+		} while (blcontinue != null && toReturn.size() < depth);
 
 		return toReturn;
 	}
@@ -705,26 +704,28 @@ public ArrayList<PageLocation> getAllPages(String language, int depth, String fr
 
 		do {
 			//Make query call.
+			int pagesNeeded = toReturn.size() - depth;
+			int batchSize = Math.min(APIlimit, pagesNeeded);
 			String serverOutput;
 			if (apnamespace == null) {
 				if (apcontinue == null) {
 					if (from != null) {
-						serverOutput = APIcommand(new QueryAllPages(language, APIlimit, from));
+						serverOutput = APIcommand(new QueryAllPages(language, batchSize, from));
 					} else {
-						serverOutput = APIcommand(new QueryAllPages(language, APIlimit));
+						serverOutput = APIcommand(new QueryAllPages(language, batchSize));
 					}
 				} else {
-					serverOutput = APIcommand(new QueryAllPages(language, APIlimit, apcontinue));
+					serverOutput = APIcommand(new QueryAllPages(language, batchSize, apcontinue));
 				}
 			} else {
 				if (apcontinue == null) {
 					if (from != null) {
-						serverOutput = APIcommand(new QueryAllPages(language, APIlimit, from, apnamespace));
+						serverOutput = APIcommand(new QueryAllPages(language, batchSize, from, apnamespace));
 					} else {
-						serverOutput = APIcommand(new QueryAllPages(language, APIlimit, apnamespace));
+						serverOutput = APIcommand(new QueryAllPages(language, batchSize, apnamespace));
 					}
 				} else {
-					serverOutput = APIcommand(new QueryAllPages(language, APIlimit, apcontinue, apnamespace));
+					serverOutput = APIcommand(new QueryAllPages(language, batchSize, apcontinue, apnamespace));
 				}
 			}
 
@@ -757,7 +758,7 @@ public ArrayList<PageLocation> getAllPages(String language, int depth, String fr
 			} else {
 				apcontinue = null;
 			}
-		} while (apcontinue != null && toReturn.size() != depth);
+		} while (apcontinue != null && toReturn.size() < depth);
 
 		return toReturn;
 	}
@@ -1188,7 +1189,7 @@ public ArrayList<ArrayList<Revision>> getUserContribs(ArrayList<User> users, int
 		logFine(userLogMessage);
 
 		//Method code below
-		ArrayList<ArrayList<Revision>> multiContribs = new ArrayList<ArrayList<Revision>>();
+		ArrayList<ArrayList<Revision>> multiContribs = new ArrayList<>();
 
 		String language = users.get(0).getLanguage();
 
@@ -1206,17 +1207,13 @@ public ArrayList<ArrayList<Revision>> getUserContribs(ArrayList<User> users, int
 			User user = users.get(u);
 
 			//Query user's contributions.
-			int rev = 0;
+			multiContribs.add(new ArrayList<Revision>());
 			boolean moreRevisionsExist = true;
 			String queryContinue = null; // User for continuing queries.
-			while (rev < depth && moreRevisionsExist) {
+			while (multiContribs.get(u).size() < depth && moreRevisionsExist) {
 				//Query the server.
-				int querySize = -1;
-				if (rev + maxQuerySize < depth) {
-					querySize = maxQuerySize;
-				} else {
-					querySize = depth - rev;
-				}
+				int querySize = Math.min(maxQuerySize, depth - multiContribs.get(u).size());
+
 				APIcommand queryUserContribs = new QueryUserContribs(user, properties, querySize);
 				if (queryContinue != null) {
 					queryUserContribs.addParameter("ucstart", queryContinue);
@@ -1236,8 +1233,6 @@ public ArrayList<ArrayList<Revision>> getUserContribs(ArrayList<User> users, int
 				}
 
 				//Parse Json for contribs
-				ArrayList<Revision> contribs = new ArrayList<Revision>();
-				
 				JsonNode queryNode = rootNode.findValue("query");
 				JsonNode userContribs = queryNode.findValue("usercontribs");
 
@@ -1265,13 +1260,9 @@ public ArrayList<ArrayList<Revision>> getUserContribs(ArrayList<User> users, int
 					}
 
 					//Package and ship the revision! Then have it sink due to a bunyip and cucumber sandwiches.
-					contribs.add(new Revision(loc, userName, comment, date, flags));
+					multiContribs.get(u).add(new Revision(loc, userName, comment, date, flags));
 				}
 
-				multiContribs.add(contribs);
-				
-				rev += querySize;
-				
 				//Parse for query continue
 				JsonNode queryContinueNode = rootNode.findValue("query-continue");
 				if (queryContinueNode == null) {
@@ -1658,13 +1649,13 @@ private String APIcommandHTTP(APIcommand command) {
 			}
 		}
 
-		//Follor the url!
+		//Follow the url!
 		try {
-			String[] output = getURL(url, command.shouldUnescapeHTML());
+			String output = removeBOM(EntityUtils.toString(getURL(url)));
 			if (output == null) {
 				throw new NetworkError("Cannot connect to server at: " + baseURL);
 			} else {
-				return ArrayUtils.compactArray(output, "\n");
+				return output;
 			}
 		} catch (IOException e) {
 			throw new Error(e);

src/main/java/WikiBot/Core/NetworkingBase.java

@@ -1,10 +1,6 @@
 package WikiBot.Core;
 
-import org.apache.commons.lang3.StringEscapeUtils;
-
-import java.io.BufferedReader;
 import java.io.IOException;
-import java.io.InputStreamReader;
 import java.io.UnsupportedEncodingException;
 import java.net.HttpURLConnection;
 import java.net.MalformedURLException;
@@ -14,21 +10,31 @@
 import java.net.URLEncoder;
 import java.net.UnknownHostException;
 import java.nio.charset.StandardCharsets;
+import java.security.KeyManagementException;
+import java.security.NoSuchAlgorithmException;
+import java.security.SecureRandom;
+import java.security.cert.CertificateException;
+import java.security.cert.X509Certificate;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.logging.Level;
 
+import javax.net.ssl.SSLContext;
 import javax.net.ssl.SSLHandshakeException;
+import javax.net.ssl.X509TrustManager;
 
-import org.apache.http.Header;
 import org.apache.http.HttpEntity;
 import org.apache.http.HttpResponse;
 import org.apache.http.NameValuePair;
 import org.apache.http.NoHttpResponseException;
 import org.apache.http.client.HttpClient;
 import org.apache.http.client.entity.UrlEncodedFormEntity;
+import org.apache.http.client.methods.HttpGet;
 import org.apache.http.client.methods.HttpPost;
 import org.apache.http.client.protocol.HttpClientContext;
+import org.apache.http.conn.ssl.NoopHostnameVerifier;
+import org.apache.http.conn.ssl.SSLConnectionSocketFactory;
+import org.apache.http.ssl.SSLContexts;
 import org.apache.http.cookie.Cookie;
 import org.apache.http.impl.client.HttpClientBuilder;
 import org.apache.http.message.BasicNameValuePair;
@@ -65,9 +71,53 @@ public class NetworkingBase extends javax.swing.JPanel {
 	//Special characters.
 	private static final String UTF8_BOM = "\uFEFF";
 
+		// One-off SSL trust manager class.
+		public class HttpsTrustManager implements X509TrustManager {
+			@Override
+			public void checkClientTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
+				// TODO Auto-generated method stub
+				
+			}
+	
+			@Override
+			public void checkServerTrusted(X509Certificate[] arg0, String arg1) throws CertificateException {
+				// TODO Auto-generated method stub
+				
+			}
+	
+			@Override
+			public X509Certificate[] getAcceptedIssuers() {
+				// TODO Auto-generated method stub
+				return new X509Certificate[]{};
+			}
+	
+		}
+	
+	private void setupSSLclient() {
+		// Handle SSL
+		SSLContext sslcontext = null;
+		SSLConnectionSocketFactory factory = null;
+
+			
+			try {
+				sslcontext = SSLContexts.custom().useProtocol("SSL").build();
+				sslcontext.init(null, new X509TrustManager[]{new HttpsTrustManager()}, new SecureRandom());
+		        factory = new SSLConnectionSocketFactory(sslcontext, new NoopHostnameVerifier());
+			} catch (KeyManagementException | NoSuchAlgorithmException e) {
+				// TODO Auto-generated catch block
+				logError("Could not create SSL context. Entering fail state.");
+				throw new Error(e.getMessage());
+			}
+
+
+			    
+		// Create client and context for web stuff.
+		httpclient = HttpClientBuilder.create().setSSLSocketFactory(factory).build();
+	}
+		
 	//Instantiation.
 	public NetworkingBase() {
-		httpclient = HttpClientBuilder.create().build();
+		setupSSLclient();
 		context =  HttpClientContext.create();
 	}
 
@@ -187,38 +237,22 @@ public void setLogPropagation(boolean set) {
 	 * @param ur The url you want to get.
 	 * @param unescapeHTML4 Unescapes HTML4 text. Ex: & #039;
 	 */
-	protected String[] getURL(String ur, boolean unescapeHTML4) throws IOException {
+	protected HttpEntity getURL(String ur) throws IOException {
 		logFiner("Loading: " + ur);
+	  		
+  		//This method actual fetches a web page, and turns it into a more easily use-able format.		  		//This method actual fetches a web page, and turns it into a more easily use-able format.
+		HttpResponse response = null;
+	 	HttpGet httpost = new HttpGet(ur);
 
-		//This method actual fetches a web page, and turns it into a more easily use-able format.
-        URL oracle = null;
+		// Fetch the url.
 		try {
-			oracle = new URL(ur);
-		} catch (MalformedURLException e) {
-			System.err.println(e.getMessage());
+			response = httpclient.execute(httpost, context);
+		} catch (SocketException|NoHttpResponseException e) {
+			throw new NetworkError("Cannot connect to server at: " + ur);
+		} catch (IOException e) {
+			e.printStackTrace();
 		}
-
-		BufferedReader in = null;
-        try {
-            in = new BufferedReader(new InputStreamReader(oracle.openStream(), StandardCharsets.UTF_8));
-        } catch (IOException e) {
-        	logError("Connection cannot be opened.");
-        	return null;
-        }
-        
-        ArrayList<String> page = new ArrayList<String>();
-        String inputLine;
-        while ((inputLine = in.readLine()) != null) {
-        	if (unescapeHTML4) {
-        		inputLine = StringEscapeUtils.unescapeHtml4(StringEscapeUtils.unescapeHtml4(inputLine));
-        	}
-        	inputLine = removeBOM(inputLine);
-        	
-            page.add(inputLine);
-        }
-        in.close();
-        
-        return page.toArray(new String[page.size()]);
+		return response.getEntity();
 	}
 
 	/*