From 8b0085b2821e38472c9fc3a44fe012717236d40d Mon Sep 17 00:00:00 2001 From: Abbe98 Date: Thu, 7 Oct 2021 16:32:57 +0200 Subject: [PATCH 01/18] update wiki-java Updating Wiki-java appears to resolve #138. --- src/org/wikipedia/Wiki.java | 7990 ++++++++++++++++++++--------------- src/pattypan/Main.java | 2 +- src/pattypan/Session.java | 2 +- 3 files changed, 4468 insertions(+), 3526 deletions(-) diff --git a/src/org/wikipedia/Wiki.java b/src/org/wikipedia/Wiki.java index c378f0f..14aa8d2 100644 --- a/src/org/wikipedia/Wiki.java +++ b/src/org/wikipedia/Wiki.java @@ -1,6 +1,6 @@ /** - * @(#)Wiki.java 0.33 10/08/2017 - * Copyright (C) 2007 - 2017 MER-C and contributors + * @(#)Wiki.java 0.36 08/02/2019 + * Copyright (C) 2007 - 2019 MER-C and contributors * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -22,45 +22,65 @@ import java.io.*; import java.net.*; +import java.net.http.*; import java.nio.*; +import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.text.Normalizer; import java.time.*; import java.time.format.*; +import java.time.temporal.ChronoUnit; import java.util.*; +import java.util.concurrent.*; import java.util.function.*; import java.util.logging.*; -import java.util.stream.Collectors; +import java.util.stream.*; import java.util.zip.GZIPInputStream; import javax.security.auth.login.*; /** * This is a somewhat sketchy bot framework for editing MediaWiki wikis. - * Requires JDK 1.8 or greater. Uses the MediaWiki API for most * operations. It is recommended that the server runs the latest version - * of MediaWiki (1.31), otherwise some functions may not work. + * of MediaWiki (1.31), otherwise some functions may not work. This framework + * requires no dependencies outside the core JDK and does not implement any + * functionality added by MediaWiki extensions. *

* Extended documentation is available * here. * All wikilinks are relative to the English Wikipedia and all timestamps are in * your wiki's time zone. - *

+ *

* Please file bug reports here - * or at the Github issue - * tracker. + * or at the Github issue + * tracker. + * + *

Configuration variables

+ *

+ * Some configuration is available through java.util.Properties. + * Set the system property wiki-java.properties to a file path + * where a configuration file is located. The available variables are: + *

* * @author MER-C and contributors - * @version 0.33 + * @version 0.36 */ -public class Wiki implements Serializable +public class Wiki implements Comparable { - // Master TODO list: - // *Admin stuff - // *More multiqueries - // *Generators (hard) - // NAMESPACES /** @@ -264,6 +284,12 @@ public class Wiki implements Serializable */ public static final String PATROL_LOG = "patrol"; + /** + * Denotes the page creation log. + * @since 0.36 + */ + public static final String PAGE_CREATION_LOG = "create"; + // PROTECTION LEVELS /** @@ -273,14 +299,14 @@ public class Wiki implements Serializable public static final String NO_PROTECTION = "all"; /** - * Denotes semi-protection (i.e. only autoconfirmed users can perform a - * particular action). + * Denotes semi-protection (only autoconfirmed users can perform a + * action). * @since 0.09 */ public static final String SEMI_PROTECTION = "autoconfirmed"; /** - * Denotes full protection (i.e. only admins can perfom a particular action). + * Denotes full protection (only admins can perfom a particular action). * @since 0.09 */ public static final String FULL_PROTECTION = "sysop"; @@ -288,21 +314,21 @@ public class Wiki implements Serializable // ASSERTION MODES /** - * Use no assertions (i.e. 0). + * Use no assertions. * @see #setAssertionMode * @since 0.11 */ public static final int ASSERT_NONE = 0; /** - * Assert that we are logged in (i.e. 1). This is checked every action. + * Assert that we are logged in. This is checked every action. * @see #setAssertionMode * @since 0.30 */ public static final int ASSERT_USER = 1; /** - * Assert that we have a bot flag (i.e. 2). This is checked every action. + * Assert that we have a bot flag. This is checked every action. * @see #setAssertionMode * @since 0.11 */ @@ -317,74 +343,41 @@ public class Wiki implements Serializable public static final int ASSERT_NO_MESSAGES = 4; /** - * Assert that we have a sysop flag (i.e. 8). This is checked intermittently. + * Assert that we have a sysop flag. This is checked intermittently. * @see #setAssertionMode * @since 0.30 */ public static final int ASSERT_SYSOP = 8; - // RC OPTIONS - - /** - * In queries against the recent changes table, this would mean we don't - * fetch anonymous edits. - * @since 0.20 - */ - public static final int HIDE_ANON = 1; - - /** - * In queries against the recent changes table, this would mean we don't - * fetch edits made by bots. - * @since 0.20 - */ - public static final int HIDE_BOT = 2; - - /** - * In queries against the recent changes table, this would mean we don't - * fetch by the logged in user. - * @since 0.20 - */ - public static final int HIDE_SELF = 4; - - /** - * In queries against the recent changes table, this would mean we don't - * fetch minor edits. - * @since 0.20 - */ - public static final int HIDE_MINOR = 8; - - /** - * In queries against the recent changes table, this would mean we don't - * fetch patrolled edits. - * @since 0.20 - */ - public static final int HIDE_PATROLLED = 16; - // REVISION OPTIONS /** - * In Revision.diff(), denotes the next revision. - * @see org.wikipedia.Wiki.Revision#diff(org.wikipedia.Wiki.Revision) + * In {@link org.wikipedia.Wiki.Revision#diff(long) Revision.diff()}, + * denotes the next revision. + * @see org.wikipedia.Wiki.Revision#diff(long) * @since 0.21 */ public static final long NEXT_REVISION = -1L; /** - * In Revision.diff(), denotes the current revision. - * @see org.wikipedia.Wiki.Revision#diff(org.wikipedia.Wiki.Revision) + * In {@link org.wikipedia.Wiki.Revision#diff(long) Revision.diff()}, + * denotes the current revision. + * @see org.wikipedia.Wiki.Revision#diff(long) * @since 0.21 */ public static final long CURRENT_REVISION = -2L; /** - * In Revision.diff(), denotes the previous revision. - * @see org.wikipedia.Wiki.Revision#diff(org.wikipedia.Wiki.Revision) + * In {@link org.wikipedia.Wiki.Revision#diff(long) Revision.diff()}, + * denotes the previous revision. + * @see org.wikipedia.Wiki.Revision#diff(long) * @since 0.21 */ public static final long PREVIOUS_REVISION = -3L; /** * The list of options the user can specify for his/her gender. + * @see User#getGender() * @since 0.24 */ public enum Gender @@ -412,230 +405,290 @@ public enum Gender unknown; } - private static final String version = "0.33"; + private static final String version = "0.36"; + + // fundamental URL strings + private final String protocol, domain, scriptPath; + private String base, articleUrl; + + /** + * Stores default HTTP parameters for API calls. Contains {@linkplain + * #setMaxLag(int) maxlag}, {@linkplain #setResolveRedirects(boolean) redirect + * resolution} and {@linkplain #setAssertionMode(int) user and bot assertions} + * when wanted by default. Add stuff to this map if you want to add parameters + * to every API call. + * @see #makeApiCall(Map, Map, String) + */ + protected ConcurrentHashMap defaultApiParams; + + /** + * URL entrypoint for the MediaWiki API. (Needs to be accessible to + * subclasses.) + * @see #initVars() + * @see #getApiUrl() + * @see MediaWiki + * documentation + */ + protected String apiUrl; - // the domain of the wiki - private String domain; - protected String query, base, apiUrl; - protected String scriptPath = "/w"; + // wiki properties + private boolean siteinfofetched = false; private boolean wgCapitalLinks = true; - private ZoneId timezone = ZoneId.of("UTC"); + private String dbname; + private String mwVersion; + private ZoneId timezone = ZoneOffset.UTC; + private Locale locale = Locale.ENGLISH; + private List extensions = Collections.emptyList(); + private LinkedHashMap namespaces = null; + private ArrayList ns_subpages = null; // user management - private Map cookies = new HashMap<>(12); + private HttpClient client; + private final CookieManager cookies; private User user; private int statuscounter = 0; - // various caches - private transient LinkedHashMap namespaces = null; - private transient ArrayList ns_subpages = null; - private transient List watchlist = null; + // watchlist cache + private List watchlist = null; // preferences private int max = 500; private int slowmax = 50; - private int throttle = 10000; // throttle + private int throttle = 10000; private int maxlag = 5; private int assertion = ASSERT_NONE; // assertion mode - private transient int statusinterval = 100; // status check + private int statusinterval = 100; // status check private int querylimit = Integer.MAX_VALUE; private String useragent = "Wiki.java/" + version + " (https://github.com/MER-C/wiki-java/)"; private boolean zipped = true; private boolean markminor = false, markbot = false; private boolean resolveredirect = false; - private String protocol = "https://"; private Level loglevel = Level.ALL; private static final Logger logger = Logger.getLogger("wiki"); // Store time when the last throttled action was executed private long lastThrottleActionTime = 0; - // retry count - private int maxtries = 2; - - // serial version - private static final long serialVersionUID = -8745212681497643456L; - - // time to open a connection - private static final int CONNECTION_CONNECT_TIMEOUT_MSEC = 30000; // 30 seconds - // time for the read to take place. (needs to be longer, some connections are slow - // and the data volume is large!) - private static final int CONNECTION_READ_TIMEOUT_MSEC = 180000; // 180 seconds - // log2(upload chunk size). Default = 22 => upload size = 4 MB. Disable - // chunked uploads by setting a large value here (50 = 1 PB will do). - private static final int LOG2_CHUNK_SIZE = 22; - // maximum URL length in bytes - protected static final int URL_LENGTH_LIMIT = 8192; + // config via properties + private final int maxtries; + private final int read_timeout_msec; + private final int log2_upload_size; // CONSTRUCTORS AND CONFIGURATION /** - * Creates a new connection to the English Wikipedia via HTTPS. - * @since 0.02 - * @deprecated use Wiki#createInstance instead - */ - @Deprecated - public Wiki() - { - this("en.wikipedia.org", "/w"); - } - - /** - * Creates a new connection to a wiki via HTTPS. WARNING: if the wiki uses - * a $wgScriptpath other than the default /w, you need to call - * getScriptPath() to automatically set it. Alternatively, you - * can use the constructor below if you know it in advance. - * - * @param domain the wiki domain name e.g. en.wikipedia.org (defaults to - * en.wikipedia.org) - * @deprecated use Wiki#createInstance instead - */ - @Deprecated - public Wiki(String domain) - { - this(domain, "/w"); - } - - /** - * Creates a new connection to a wiki with $wgScriptpath set to - * scriptPath via HTTPS. - * - * @param domain the wiki domain name - * @param scriptPath the script path - * @since 0.14 - * @deprecated use Wiki#createInstance instead - */ - @Deprecated - public Wiki(String domain, String scriptPath) - { - this(domain, scriptPath, "https://"); - } - - /** - * Creates a new connection to a wiki with $wgScriptpath set to - * scriptPath via the specified protocol. + * Creates a new MediaWiki API client for the given wiki with + * $wgScriptPath set to scriptPath and via the + * specified protocol. * * @param domain the wiki domain name * @param scriptPath the script path * @param protocol a protocol e.g. "http://", "https://" or "file:///" * @since 0.31 - * @deprecated use Wiki#createInstance instead; this will be made private - * for the reason in the TODO comment below */ - @Deprecated - public Wiki(String domain, String scriptPath, String protocol) - { - if (domain == null || domain.isEmpty()) - domain = "en.wikipedia.org"; - this.domain = domain; - this.scriptPath = scriptPath; - this.protocol = protocol; - - // init variables - // This is fine as long as you do not have parameters other than domain - // and scriptpath in constructors and do not do anything else than super(x)! - // http://stackoverflow.com/questions/3404301/whats-wrong-with-overridable-method-calls-in-constructors - // TODO: remove this + protected Wiki(String domain, String scriptPath, String protocol) + { + this.domain = Objects.requireNonNull(domain); + this.scriptPath = Objects.requireNonNull(scriptPath); + this.protocol = Objects.requireNonNull(protocol); + + defaultApiParams = new ConcurrentHashMap<>(); + defaultApiParams.put("format", "xml"); + defaultApiParams.put("maxlag", String.valueOf(maxlag)); + logger.setLevel(loglevel); logger.log(Level.CONFIG, "[{0}] Using Wiki.java {1}", new Object[] { domain, version }); - initVars(); + + // read in config + Properties props = new Properties(); + String filename = System.getProperty("wiki-java.properties"); + if (filename != null) + { + try + { + InputStream in = new FileInputStream(new File(filename)); + props.load(in); + } + catch (IOException ex) + { + logger.log(Level.WARNING, "Unable to load properties file " + filename); + } + } + maxtries = Integer.parseInt(props.getProperty("maxretries", "2")); + log2_upload_size = Integer.parseInt(props.getProperty("loguploadsize", "22")); // 4 MB + read_timeout_msec = Integer.parseInt(props.getProperty("readtimeout", "180000")); // 180 seconds + cookies = new CookieManager(null, CookiePolicy.ACCEPT_ALL); + client = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(30)) + .cookieHandler(cookies) + .build(); } - + /** - * Creates a new connection to a wiki via HTTPS. Depending on the settings - * of the wiki, you may need to call {@link Wiki#getSiteInfo()} on the - * returned object after this in order for some functionality to work - * correctly. + * Creates a new MediaWiki API client for the given wiki using HTTPS. + * Depending on the settings of the wiki, you may need to call {@link + * Wiki#getSiteInfo()} on the returned object after this in order for some + * functionality to work correctly. * * @param domain the wiki domain name e.g. en.wikipedia.org (defaults to * en.wikipedia.org) - * @return the created wiki + * @return the constructed API client object * @since 0.34 */ - public static Wiki createInstance(String domain) + public static Wiki newSession(String domain) { - return createInstance(domain, "/w", "https://"); + return newSession(domain, "/w", "https://"); } - + /** - * Creates a new connection to a wiki with - * $wgScriptPath set to scriptPath and via the - * specified protocol. Depending on the settings of the wiki, you may need - * to call {@link Wiki#getSiteInfo()} on the returned object after this in + * $wgScriptPath set to scriptPath and via the + * specified protocol. Depending on the settings of the wiki, you may need + * to call {@link Wiki#getSiteInfo()} on the returned object after this in * order for some functionality to work correctly. * + *

All factory methods in subclasses must call {@link #initVars()}. + * * @param domain the wiki domain name * @param scriptPath the script path * @param protocol a protocol e.g. "http://", "https://" or "file:///" - * @return the constructed Wiki object + * @return the constructed API client object * @since 0.34 */ - public static Wiki createInstance(String domain, String scriptPath, String protocol) + public static Wiki newSession(String domain, String scriptPath, String protocol) { // Don't put network requests here. Servlets cannot afford to make // unnecessary network requests in initialization. - Wiki wiki = new Wiki(domain, "/w", protocol); - wiki.initVars(); // construct URL bases + Wiki wiki = new Wiki(domain, scriptPath, protocol); + wiki.initVars(); return wiki; } /** * Edit this if you need to change the API and human interface url - * configuration of the wiki. One example use is server-side cache - * management (maxage and smaxage API parameters). + * configuration of the wiki. One example use is to change the port number. * *

Contributed by Tedder * @since 0.24 */ protected void initVars() { - StringBuilder basegen = new StringBuilder(protocol); - basegen.append(domain); - basegen.append(scriptPath); - StringBuilder apigen = new StringBuilder(basegen); - apigen.append("/api.php?format=xml&"); - // MediaWiki has inbuilt maxlag functionality, see [[mw:Manual:Maxlag - // parameter]]. Let's exploit it. - if (maxlag >= 0) - { - apigen.append("maxlag="); - apigen.append(maxlag); - apigen.append("&"); - basegen.append("/index.php?maxlag="); - basegen.append(maxlag); - basegen.append("&title="); - } - else - basegen.append("/index.php?title="); - base = basegen.toString(); - // the native API supports assertions as of MW 1.23 - if ((assertion & ASSERT_BOT) == ASSERT_BOT) - apigen.append("assert=bot&"); - else if ((assertion & ASSERT_USER) == ASSERT_USER) - apigen.append("assert=user&"); - apiUrl = apigen.toString(); - apigen.append("action=query&"); - if (resolveredirect) - apigen.append("redirects&"); - query = apigen.toString(); + base = protocol + domain + scriptPath + "/index.php"; + apiUrl = protocol + domain + scriptPath + "/api.php"; + articleUrl = protocol + domain + "/wiki/"; } /** - * Gets the domain of the wiki, as supplied on construction. + * Gets the domain of the wiki as supplied on construction. * @return the domain of the wiki * @since 0.06 */ - public String getDomain() + public final String getDomain() { return domain; } - - public String getProtocol() + + /** + * Gets the + * $wgScriptPath variable as supplied on construction. + * @return the script path of the wiki + * @since 0.14 + */ + public final String getScriptPath() + { + return scriptPath; + } + + /** + * Gets the protocol used to access this MediaWiki instance, as supplied + * on construction. + * @return (see above) + * @since 0.35 + */ + public final String getProtocol() { return protocol; } + /** + * Determines whether this wiki is equal to another object based on the + * protocol (not case sensitive), domain (not case sensitive) and + * scriptPath (case sensitive). A return value of {@code true} means two + * Wiki objects point to the same instance of MediaWiki. + * @param obj the object to compare + * @return whether the wikis point to the same instance of MediaWiki + */ + @Override + public boolean equals(Object obj) + { + if (!(obj instanceof Wiki)) + return false; + Wiki other = (Wiki)obj; + return domain.equalsIgnoreCase(other.domain) + && scriptPath.equals(other.scriptPath) + && protocol.equalsIgnoreCase(other.protocol); + } + + /** + * Returns a hash code of this object based on the protocol, domain and + * scriptpath. + * @return a hash code + */ + @Override + public int hashCode() + { + // English locale used here for reproducability and so network requests + // are not required + int hc = domain.toLowerCase(Locale.ENGLISH).hashCode(); + hc = 127 * hc + scriptPath.hashCode(); + hc = 127 * hc + protocol.toLowerCase(Locale.ENGLISH).hashCode(); + return hc; + } + + /** + * Allows wikis to be sorted based on their domain (case insensitive), then + * their script path (case sensitive). If 0 is returned, it is reasonable + * both Wikis point to the same instance of MediaWiki. + * @param other the wiki to compare to + * @return -1 if this wiki is alphabetically before the other, 1 if after + * and 0 if they are likely to be the same instance of MediaWiki + * @since 0.35 + */ + @Override + public int compareTo(Wiki other) + { + int result = domain.compareToIgnoreCase(other.domain); + if (result == 0) + result = scriptPath.compareTo(other.scriptPath); + return result; + } + + /** + * Gets the URL of index.php. + * @return (see above) + * @see + * MediaWiki documentation + * @since 0.35 + */ + public String getIndexPhpUrl() + { + return base; + } + + /** + * Gets the URL of api.php. + * @return (see above) + * @see MediaWiki + * documentation + * @since 0.36 + */ + public String getApiUrl() + { + return apiUrl; + } + /** * Gets the editing throttle. * @return the throttle value in milliseconds @@ -648,8 +701,11 @@ public int getThrottle() } /** - * Sets the editing throttle. Read requests are not throttled or restricted - * in any way. Default is 10s. + * Sets the throttle, which limits most write requests to no more than one + * per wiki instance in the given time across all threads. (As a + * consequence, all throttled methods are thread safe.) Read requests are + * not throttled or restricted in any way. Default is 10 seconds. + * * @param throttle the new throttle value in milliseconds * @see #getThrottle * @since 0.09 @@ -662,69 +718,166 @@ public void setThrottle(int throttle) /** * Gets various properties of the wiki and sets the bot framework up to use - * them. Also populates the namespace cache. Returns: + * them. The return value is cached. This method is thread safe. Returns: *

    *
  • usingcapitallinks: (Boolean) whether a wiki forces upper case * for the title. Example: en.wikipedia = true, en.wiktionary = false. * Default = true. See * $wgCapitalLinks - *
  • scriptpath: (String) the scriptpath: (String) the - * $wgScriptPath wiki variable. Default = /w. + * $wgScriptPath wiki variable. Default = {@code /w}. *
  • version: (String) the MediaWiki version used for this wiki - *
  • timezone: (String) the timezone the wiki is in, default = UTC + *
  • timezone: (ZoneId) the timezone the wiki is in, default = UTC + *
  • locale: (Locale) the locale of the wiki + *
  • dbname: (String) the internal name of the database *
* * @return (see above) * @since 0.30 * @throws IOException if a network error occurs + * @deprecated This method is likely going to get renamed with the return + * type changed to void once I finish cleaning up the site info caching + * mechanism. Use the specialized methods instead. */ - public Map getSiteInfo() throws IOException - { - Map ret = new HashMap<>(); - String line = fetch(query + "action=query&meta=siteinfo&siprop=namespaces%7Cnamespacealiases%7Cgeneral", "getSiteInfo"); - - // general site info - String bits = line.substring(line.indexOf("")); - wgCapitalLinks = parseAttribute(bits, "case", 0).equals("first-letter"); - ret.put("usingcapitallinks", wgCapitalLinks); - scriptPath = parseAttribute(bits, "scriptpath", 0); - ret.put("scriptpath", scriptPath); - timezone = ZoneId.of(parseAttribute(bits, "timezone", 0)); - ret.put("timezone", timezone); - ret.put("version", parseAttribute(bits, "generator", 0)); - - // populate namespace cache - namespaces = new LinkedHashMap<>(30); - ns_subpages = new ArrayList<>(30); - // xml form: Media or - String[] items = line.split("') + 1; - int c = items[i].indexOf(""); - if (c < 0) - namespaces.put("", ns); - else - namespaces.put(normalize(decode(items[i].substring(b, c))), ns); - - String canonicalnamespace = parseAttribute(items[i], "canonical", 0); - if (canonicalnamespace != null) - namespaces.put(canonicalnamespace, ns); - - // does this namespace support subpages? - if (items[i].contains("subpages=\"\"")) - ns_subpages.add(ns); + @Deprecated + public synchronized Map getSiteInfo() throws IOException + { + Map siteinfo = new HashMap<>(); + if (!siteinfofetched) + { + Map getparams = new HashMap<>(); + getparams.put("action", "query"); + getparams.put("meta", "siteinfo"); + getparams.put("siprop", "namespaces|namespacealiases|general|extensions"); + String line = makeApiCall(getparams, null, "getSiteInfo"); + detectUncheckedErrors(line, null, null); + + // general site info + String bits = line.substring(line.indexOf("")); + wgCapitalLinks = parseAttribute(bits, "case", 0).equals("first-letter"); + timezone = ZoneId.of(parseAttribute(bits, "timezone", 0)); + mwVersion = parseAttribute(bits, "generator", 0); + locale = new Locale(parseAttribute(bits, "lang", 0)); + dbname = parseAttribute(bits, "wikiid", 0); + + // parse extensions + bits = line.substring(line.indexOf(""), line.indexOf("")); + extensions = new ArrayList<>(); + String[] unparsed = bits.split("(30); + ns_subpages = new ArrayList<>(30); + // xml form: Media or + String[] items = line.split("') + 1; + int c = items[i].indexOf(""); + if (c < 0) + namespaces.put("", ns); + else + namespaces.put(normalize(decode(items[i].substring(b, c))), ns); + + String canonicalnamespace = parseAttribute(items[i], "canonical", 0); + if (canonicalnamespace != null) + namespaces.put(canonicalnamespace, ns); + + // does this namespace support subpages? + if (items[i].contains("subpages=\"\"")) + ns_subpages.add(ns); + } + siteinfofetched = true; + log(Level.INFO, "getSiteInfo", "Successfully retrieved site info for " + getDomain()); } - - initVars(); - log(Level.INFO, "getSiteInfo", "Successfully retrieved site info for " + getDomain()); - return ret; + siteinfo.put("usingcapitallinks", wgCapitalLinks); + siteinfo.put("scriptpath", scriptPath); + siteinfo.put("timezone", timezone); + siteinfo.put("version", mwVersion); + siteinfo.put("locale", locale); + siteinfo.put("extensions", extensions); + siteinfo.put("dbname", dbname); + return siteinfo; + } + + /** + * Gets the version of MediaWiki this wiki runs e.g. 1.20wmf5 (54b4fcb). + * See [[Special:Version]] on your wiki. + * @return (see above) + * @throws UncheckedIOException if the site info cache has not been + * populated and a network error occurred when populating it + * @since 0.14 + * @see MediaWiki Git + */ + public String version() + { + ensureNamespaceCache(); + return mwVersion; + } + + /** + * Detects whether a wiki forces upper case for the first character in a + * title. Example: en.wikipedia = true, en.wiktionary = false. + * @return (see above) + * @throws UncheckedIOException if the site info cache has not been + * populated and a network error occurred when populating it + * @see MediaWiki + * documentation + * @since 0.30 + */ + public boolean usesCapitalLinks() + { + ensureNamespaceCache(); + return wgCapitalLinks; + } + + /** + * Returns the list of extensions installed on this wiki. + * @return (see above) + * @throws UncheckedIOException if the site info cache has not been + * populated and a network error occurred when populating it + * @see MediaWiki + * documentation + * @since 0.35 + */ + public List installedExtensions() + { + ensureNamespaceCache(); + return new ArrayList<>(extensions); + } + + /** + * Gets the timezone of this wiki + * @return (see above) + * @throws UncheckedIOException if the site info cache has not been + * populated and a network error occurred when populating it + * @since 0.35 + */ + public ZoneId timezone() + { + ensureNamespaceCache(); + return timezone; + } + + /** + * Gets the locale of this wiki. + * @return (see above) + * @throws UncheckedIOException if the site info cache has not been + * populated and a network error occurred when populating it + * @since 0.35 + */ + public Locale locale() + { + ensureNamespaceCache(); + return locale; } /** @@ -753,7 +906,9 @@ public String getUserAgent() * Enables/disables GZip compression for GET requests. Default: true. * @param zipped whether we use GZip compression * @since 0.23 + * @deprecated this is now handled transparently; just delete calls to this method. */ + @Deprecated(forRemoval=true) public void setUsingCompressedRequests(boolean zipped) { this.zipped = zipped; @@ -764,7 +919,9 @@ public void setUsingCompressedRequests(boolean zipped) * Default: true. * @return (see above) * @since 0.23 + * @deprecated this is now handled transparently; just delete calls to this method. */ + @Deprecated(forRemoval=true) public boolean isUsingCompressedRequests() { return zipped; @@ -790,7 +947,10 @@ public boolean isResolvingRedirects() public void setResolveRedirects(boolean b) { resolveredirect = b; - initVars(); + if (b) + defaultApiParams.put("redirects", "1"); + else + defaultApiParams.remove("redirects"); } /** @@ -863,31 +1023,6 @@ public void setQueryLimit(int limit) querylimit = limit; } - /** - * Determines whether this wiki is equal to another object. - * @param obj the object to compare - * @return whether the domains of the wikis are equal - * @since 0.10 - */ - @Override - public boolean equals(Object obj) - { - if (!(obj instanceof Wiki)) - return false; - return domain.equals(((Wiki)obj).domain); - } - - /** - * Returns a hash code of this object. - * @return a hash code - * @since 0.12 - */ - @Override - public int hashCode() - { - return domain.hashCode() * maxlag - throttle; - } - /** * Returns a string representation of this Wiki. * @return a string representation of this Wiki. @@ -897,12 +1032,14 @@ public int hashCode() public String toString() { // domain - StringBuilder buffer = new StringBuilder("Wiki[domain="); + StringBuilder buffer = new StringBuilder("Wiki[url="); + buffer.append(protocol); buffer.append(domain); + buffer.append(scriptPath); // user buffer.append(",user="); - buffer.append(user != null ? user.toString() : "null"); + buffer.append(Objects.toString(user)); buffer.append(","); // throttle mechanisms @@ -948,7 +1085,10 @@ public void setMaxLag(int lag) { maxlag = lag; log(Level.CONFIG, "setMaxLag", "Setting maximum allowable database lag to " + lag); - initVars(); + if (maxlag >= 0) + defaultApiParams.put("maxlag", String.valueOf(maxlag)); + else + defaultApiParams.remove("maxlag"); } /** @@ -973,7 +1113,13 @@ public void setAssertionMode(int mode) { assertion = mode; log(Level.CONFIG, "setAssertionMode", "Set assertion mode to " + mode); - initVars(); + + if ((assertion & ASSERT_BOT) == ASSERT_BOT) + defaultApiParams.put("assert", "bot"); + else if ((assertion & ASSERT_USER) == ASSERT_USER) + defaultApiParams.put("assert", "user"); + else + defaultApiParams.remove("assert"); } /** @@ -1021,29 +1167,36 @@ public void setLogLevel(Level loglevel) // META STUFF /** - * Logs in to the wiki. This method is thread-safe. + * Logs in to the wiki. This method is thread-safe. * * @param username a username - * @param password a password (as a char[] due to JPasswordField) + * @param password a password, as a {@code char[]} for security + * reasons. Overwritten once the password is used. * @throws IOException if a network error occurs + * @throws FailedLoginException if the login failed due to an incorrect + * username or password, the requirement for an interactive login (not + * supported, use [[Special:BotPasswords]]) or some other reason * @see #logout + * @see MediaWiki + * documentation */ public synchronized void login(String username, char[] password) throws IOException, FailedLoginException { - StringBuilder buffer = new StringBuilder(500); - buffer.append("lgname="); - buffer.append(encode(username, false)); - buffer.append("&lgpassword="); - buffer.append(encode(new String(password), false)); - buffer.append("&lgtoken="); - buffer.append(encode(getToken("login"), false)); - String line = post(apiUrl + "action=login", buffer.toString(), "login"); - buffer.setLength(0); + Map getparams = new HashMap<>(); + getparams.put("action", "login"); + Map postparams = new HashMap<>(); + postparams.put("lgname", username); + postparams.put("lgpassword", new String(password)); + postparams.put("lgtoken", getToken("login")); + String line = makeApiCall(getparams, postparams, "login"); + detectUncheckedErrors(line, null, null); + Arrays.fill(password, '0'); // check for success if (line.contains("result=\"Success\"")) { - user = new User(parseAttribute(line, "lgusername", 0)); + String returned_username = parseAttribute(line, "lgusername", 0); + user = getUsers(List.of(returned_username)).get(0); boolean apihighlimit = user.isAllowedTo("apihighlimits"); if (apihighlimit) { @@ -1055,18 +1208,21 @@ public synchronized void login(String username, char[] password) throws IOExcept else if (line.contains("result=\"Failed\"")) throw new FailedLoginException("Login failed: " + parseAttribute(line, "reason", 0)); // interactive login or bot password required - else if (line.contains("result=\"Aborted\"")) + else if (line.contains("result=\"Aborted\"")) throw new FailedLoginException("Login failed: you need to use a bot password, see [[Special:Botpasswords]]."); else throw new AssertionError("Unreachable!"); } /** - * Logs in to the wiki. This method is thread-safe. + * Logs in to the wiki. This method is thread-safe. * * @param username a username * @param password a string with the password * @throws IOException if a network error occurs + * @throws FailedLoginException if the login failed due to an incorrect + * username or password, the requirement for an interactive login (not + * supported, use [[Special:Botpasswords]]) or some other reason * @see #logout */ public synchronized void login(String username, String password) throws IOException, FailedLoginException @@ -1083,7 +1239,7 @@ public synchronized void login(String username, String password) throws IOExcept */ public synchronized void logout() { - cookies.clear(); + cookies.getCookieStore().removeAll(); user = null; max = 500; slowmax = 50; @@ -1102,10 +1258,15 @@ public synchronized void logout() * @since 0.14 * @see #login * @see #logout + * @see MediaWiki + * documentation */ public synchronized void logoutServerSide() throws IOException { - fetch(apiUrl + "action=logout", "logoutServerSide"); + Map getparams = new HashMap<>(); + getparams.put("action", "logout"); + String response = makeApiCall(getparams, null, "logoutServerSide"); + detectUncheckedErrors(response, null, null); logout(); // destroy local cookies } @@ -1118,12 +1279,18 @@ public synchronized void logoutServerSide() throws IOException */ public boolean hasNewMessages() throws IOException { - String url = query + "meta=userinfo&uiprop=hasmsg"; - return fetch(url, "hasNewMessages").contains("messages=\"\""); + Map getparams = new HashMap<>(); + getparams.put("action", "query"); + getparams.put("meta", "userinfo"); + getparams.put("uiprop", "hasmsg"); + String response = makeApiCall(getparams, null, "hasNewMessages"); + detectUncheckedErrors(response, null, null); + return response.contains("messages=\"\""); } /** - * Determines the current database replication lag. + * Determines the current database replication lag. This method does not + * wait if the maxlag setting is exceeded. This method is thread safe. * @return the current database replication lag * @throws IOException if a network error occurs * @see #setMaxLag @@ -1132,12 +1299,26 @@ public boolean hasNewMessages() throws IOException * MediaWiki documentation * @since 0.10 */ - public int getCurrentDatabaseLag() throws IOException + public double getCurrentDatabaseLag() throws IOException { - String line = fetch(query + "meta=siteinfo&siprop=dbrepllag", "getCurrentDatabaseLag"); - String lag = parseAttribute(line, "lag", 0); - log(Level.INFO, "getCurrentDatabaseLag", "Current database replication lag is " + lag + " seconds"); - return Integer.parseInt(lag); + Map getparams = new HashMap<>(); + getparams.put("action", "query"); + getparams.put("meta", "siteinfo"); + getparams.put("siprop", "dbrepllag"); + + synchronized (this) + { + // bypass lag check for this request + int temp = getMaxLag(); + setMaxLag(-1); + String line = makeApiCall(getparams, null, "getCurrentDatabaseLag"); + detectUncheckedErrors(line, null, null); + setMaxLag(temp); + + String lag = parseAttribute(line, "lag", 0); + log(Level.INFO, "getCurrentDatabaseLag", "Current database replication lag is " + lag + " seconds"); + return Double.parseDouble(lag); + } } /** @@ -1152,7 +1333,12 @@ public int getCurrentDatabaseLag() throws IOException */ public Map getSiteStatistics() throws IOException { - String text = fetch(query + "meta=siteinfo&siprop=statistics", "getSiteStatistics"); + Map getparams = new HashMap<>(); + getparams.put("action", "query"); + getparams.put("meta", "siteinfo"); + getparams.put("siprop", "statistics"); + String text = makeApiCall(getparams, null, "getSiteStatistics"); + detectUncheckedErrors(text, null, null); Map ret = new HashMap<>(20); ret.put("pages", Integer.parseInt(parseAttribute(text, "pages", 0))); ret.put("articles", Integer.parseInt(parseAttribute(text, "articles", 0))); @@ -1165,10 +1351,8 @@ public Map getSiteStatistics() throws IOException } /** - * Renders the specified wiki markup by passing it to the MediaWiki - * parser through the API. (Note: this isn't implemented locally because - * I can't be stuffed porting Parser.php). One use of this method is to - * emulate the previewing functionality of the MediaWiki software. + * Renders the specified wiki markup as HTML by passing it to the MediaWiki + * parser through the API. * * @param markup the markup to parse * @return the parsed markup as HTML @@ -1177,32 +1361,99 @@ public Map getSiteStatistics() throws IOException */ public String parse(String markup) throws IOException { - // This is POST because markup can be arbitrarily large, as in the size - // of an article (over 10kb). - String response = post(apiUrl + "action=parse", "prop=text&text=" + encode(markup, false), "parse"); - int y = response.indexOf('>', response.indexOf(""); - return decode(response.substring(y, z)); + Map content = new HashMap<>(); + content.put("text", markup); + return parse(content, -1, false); } /** - * Same as {@link #parse(java.lang.String)}, but also strips out unwanted - * crap. This might be useful to subclasses. + * Parses wikitext, revisions or pages. Deleted pages and revisions to + * deleted pages are not allowed if you don't have the rights to view them. * - * @param in the string to parse - * @return that string without the crap - * @throws IOException if a network error occurs - * @since 0.14 - */ - protected String parseAndCleanup(String in) throws IOException - { - String output = parse(in); - output = output.replace("

", "").replace("

", ""); // remove paragraph tags - output = output.replace("\n", ""); // remove new lines - - // strip out the parser report, which comes at the end - int a = output.indexOf("" nocreate="" allowusertalk=""/> + for (int a = line.indexOf(" 0; a = line.indexOf(" - for (int a = line.indexOf(" 0; a = line.indexOf("", a); - String temp = line.substring(a, b); - LogEntry le = parseLogEntry(temp); - le.type = BLOCK_LOG; - le.action = "block"; - // parseLogEntries parses block target into le.user due to mw.api - // attribute name - if (le.user == null) // autoblock - le.target = "#" + parseAttribute(temp, "id", 0); - else - le.target = namespaceIdentifier(USER_NAMESPACE) + ":" + le.user.username; - // parse blocker for real - le.user = new User(parseAttribute(temp, "by", 0)); - results.add(le); - } + // find entry + int b = Math.max(line.indexOf("/>", a), line.indexOf("", a)); + String temp = line.substring(a, b); + + String blocker = parseAttribute(temp, "by", 0); + String blockeduser = parseAttribute(temp, "user", 0); + String target; + if (blockeduser == null) // autoblock + target = "#" + parseAttribute(temp, "id", 0); + else + target = namespaceIdentifier(USER_NAMESPACE) + ":" + blockeduser; + + LogEntry le = parseLogEntry(temp, blocker, BLOCK_LOG, "block", target); + results.add(le); } - catch (IOException ex) + }; + List entries = new ArrayList<>(); + if (users == null) + entries.addAll(makeListQuery("bk", getparams, null, "getBlockList", limit, parser)); + else + { + for (String bkusers : constructTitleString(users)) { - throw new UncheckedIOException(ex); + getparams.put("bkusers", bkusers); + entries.addAll(makeListQuery("bk", getparams, null, "getBlockList", limit, parser)); } - }); - - // log statement - StringBuilder logRecord = new StringBuilder("Successfully fetched IP block list "); - if (!user.isEmpty()) - { - logRecord.append(" for "); - logRecord.append(user); - } - if (start != null) - { - logRecord.append(" from "); - logRecord.append(start.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - } - if (end != null) - { - logRecord.append(" to "); - logRecord.append(end.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); } - int size = entries.size(); - logRecord.append(" ("); - logRecord.append(size); - logRecord.append(" entries)"); - log(Level.INFO, "getIPBlockList", logRecord.toString()); - return entries.toArray(new LogEntry[size]); - } - - /** - * Gets the log entries representing actions that were performed on a - * specific target. Equivalent to [[Special:Log]]. - * @param logtype what log to get (e.g. {@link #DELETION_LOG}) - * @param action what action to get (e.g. delete, undelete, etc.), use - * null to not specify one - * @param user the user performing the action. Use null not to specify - * one. - * @param target the target of the action(s). - * @throws IOException if a network error occurs - * @return the specified log entries - * @since 0.08 - */ - public LogEntry[] getLogEntries(String logtype, String action, String user, String target) throws IOException - { - return getLogEntries(logtype, action, user, target, null, null, Integer.MAX_VALUE, ALL_NAMESPACES); - } - - /** - * Gets the last how ever many log entries in the specified log. Equivalent - * to [[Special:Log]] and [[Special:Newimages]] when - * type.equals({@link #UPLOAD_LOG}). - * - * @param logtype what log to get (e.g. {@link #DELETION_LOG}) - * @param action what action to get (e.g. delete, undelete, etc.), use - * null to not specify one - * @param amount the number of entries to get (overrides global limits) - * @throws IOException if a network error occurs - * @throws IllegalArgumentException if the log type doesn't exist - * @return the specified log entries - */ - public LogEntry[] getLogEntries(String logtype, String action, int amount) throws IOException - { - return getLogEntries(logtype, action, null, null, null, null, amount, ALL_NAMESPACES); + log(Level.INFO, "getBlockList", "Successfully fetched block list " + entries.size() + " entries)"); + return entries; } /** * Gets the specified amount of log entries between the given times by * the given user on the given target. Equivalent to [[Special:Log]]. - * WARNING: the start date is the most recent of the dates given, and - * the order of enumeration is from newest to oldest. + * Accepted parameters from helper are: + * + *
    + *
  • {@link Wiki.RequestHelper#withinDateRange(OffsetDateTime, + * OffsetDateTime) date range} + *
  • {@link Wiki.RequestHelper#byUser(String) user} + *
  • {@link Wiki.RequestHelper#byTitle(String) title} + *
  • {@link Wiki.RequestHelper#reverse(boolean) reverse} + *
  • {@link Wiki.RequestHelper#inNamespaces(int...) namespaces} (one + * namespace only, must not be used if a title is specified) + *
  • {@link Wiki.RequestHelper#taggedWith(String) tag} + *
  • {@link Wiki.RequestHelper#limitedTo(int) local query limit} + *
* * @param logtype what log to get (e.g. {@link #DELETION_LOG}) - * @param action what action to get (e.g. delete, undelete, etc.), use - * null to not specify one - * @param user the user performing the action. Use null not to specify - * one. - * @param target the target of the action. Use null not to specify one. - * @param start what timestamp to start. Use null to not specify one. - * @param end what timestamp to end. Use null to not specify one. - * @param amount the amount of log entries to get. If both start and - * end are defined, this is ignored. Use Integer.MAX_VALUE to not - * specify one (overrides global limits) - * @param namespace filters by namespace. Returns empty if namespace - * doesn't exist. Use {@link #ALL_NAMESPACES} to not specify one. + * @param action what action to get (e.g. delete, undelete, etc.), use + * {@code null} to not specify one + * @param helper a {@link Wiki.RequestHelper} (optional, use null to not + * provide any of the optional parameters noted above) * @throws IOException if a network error occurs - * @throws IllegalArgumentException if start < end or amount < 1 + * @throws SecurityException if the user lacks the credentials needed to + * access a privileged log * @return the specified log entries * @since 0.08 */ - public LogEntry[] getLogEntries(String logtype, String action, String user, String target, - OffsetDateTime start, OffsetDateTime end, int amount, int namespace) throws IOException + public List getLogEntries(String logtype, String action, Wiki.RequestHelper helper) throws IOException { - // construct the query url from the parameters given - StringBuilder url = new StringBuilder(query); - url.append("list=logevents&leprop=ids%7Ctitle%7Ctype%7Cuser%7Ctimestamp%7Ccomment%7Cdetails"); - - // check for amount - if (amount < 1) - throw new IllegalArgumentException("Tried to retrieve less than one log entry!"); - + int limit = -1; + Map getparams = new HashMap<>(); + getparams.put("list", "logevents"); + getparams.put("leprop", "ids|title|type|user|timestamp|comment|parsedcomment|details|tags"); if (!logtype.equals(ALL_LOGS)) { if (action == null) - { - url.append("&letype="); - url.append(logtype); - } + getparams.put("letype", logtype); else - { - url.append("&leaction="); - url.append(logtype); - url.append("/"); - url.append(action); - } - } - if (namespace != ALL_NAMESPACES) - { - url.append("&lenamespace="); - url.append(namespace); - } - if (user != null) - { - url.append("&leuser="); - url.append(encode(user, true)); + getparams.put("leaction", logtype + "/" + action); } - if (target != null) + if (helper != null) { - url.append("&letitle="); - url.append(encode(target, true)); - } - if (start != null) - { - if (end != null && start.isBefore(end)) //aargh - throw new IllegalArgumentException("Specified start date is before specified end date!"); - url.append("&lestart="); - url.append(start.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - } - if (end != null) - { - url.append("&leend="); - url.append(end.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + helper.setRequestType("le"); + getparams.putAll(helper.addTitleParameter()); + getparams.putAll(helper.addDateRangeParameters()); + getparams.putAll(helper.addUserParameter()); + getparams.putAll(helper.addReverseParameter()); + getparams.putAll(helper.addNamespaceParameter()); + getparams.putAll(helper.addTagParameter()); + limit = helper.limit(); } - int originallimit = getQueryLimit(); - setQueryLimit(Math.min(amount, originallimit)); - List entries = queryAPIResult("le", url, "getLogEntries", (line, results) -> + List entries = makeListQuery("le", getparams, null, "getLogEntries", limit, (line, results) -> { String[] items = line.split("getLogEntries(), - * getImageHistory() and getIPBlockList() into LogEntry + * Parses xml generated by getLogEntries(), + * getImageHistory() and getBlockList() into {@link Wiki.LogEntry} * objects. Override this if you want custom log types. NOTE: if * RevisionDelete was used on a log entry, the relevant values will be * null. * * @param xml the xml to parse + * @param user null, or use this value for the performer of the log entry + * @param type null, or use this value for the type of the log entry + * @param action null, or use this value for the action of the log entry + * @param target null, or user this value for the target of the log entry * @return the parsed log entry * @since 0.18 */ - protected LogEntry parseLogEntry(String xml) + protected LogEntry parseLogEntry(String xml, String user, String type, String action, String target) { - // note that we can override these in the calling method - String type = "", action = ""; + // ID (getLogEntries only) + long id = -1; + if (xml.contains("logid=\"")) + id = Long.parseLong(parseAttribute(xml, "logid", 0)); + boolean actionhidden = xml.contains("actionhidden=\""); - if (xml.contains("type=\"")) // only getLogEntries + if (type == null && xml.contains("type=\"")) // only getLogEntries { type = parseAttribute(xml, "type", 0); action = parseAttribute(xml, "action", 0); } // reason - String reason; + String reason, parsedreason; boolean reasonhidden = xml.contains("commenthidden=\""); - if (type.equals(USER_CREATION_LOG)) // there is no reason for creating a user + if (USER_CREATION_LOG.equals(type)) + { + // there is no reason for creating a user reason = ""; + parsedreason = ""; + } else if (xml.contains("reason=\"")) + { reason = parseAttribute(xml, "reason", 0); + parsedreason = null; // not available in list=blocks / getBlockList! + } else + { reason = parseAttribute(xml, "comment", 0); + parsedreason = parseAttribute(xml, "parsedcomment", 0); + } - // generic performer name (won't work for ipblocklist, overridden there) + // generic performer name boolean userhidden = xml.contains("userhidden=\"\""); - User performer = null; - if (xml.contains("user=\"")) - performer = new User(parseAttribute(xml, "user", 0)); + if (user == null && xml.contains("user=\"")) + user = parseAttribute(xml, "user", 0); // generic target name - String target = null; - if (xml.contains(" title=\"")) // space is important -- commons.getImageHistory("File:Chief1.gif"); + // space is important -- commons.getImageHistory("File:Chief1.gif"); + if (target == null && xml.contains(" title=\"")) target = parseAttribute(xml, "title", 0); OffsetDateTime timestamp = OffsetDateTime.parse(parseAttribute(xml, "timestamp", 0)); - // details: TODO: make this a HashMap - Object details = null; + // details + Map details = new HashMap<>(); if (xml.contains("commenthidden")) // oversighted details = null; else if (type.equals(MOVE_LOG)) - details = parseAttribute(xml, "target_title", 0); // the new title + details.put("target_title", parseAttribute(xml, "target_title", 0)); else if (type.equals(BLOCK_LOG) || xml.contains(" 10) // not an unblock { int d = s.indexOf('\"', c); - details = new Object[] + if (s.contains("anononly")) // anon-only + details.put("anononly", "true"); + if (s.contains("nocreate")) // account creation blocked + details.put("nocreate", "true"); + if (s.contains("noautoblock")) // autoblock disabled + details.put("noautoblock", "true"); + if (s.contains("noemail")) // email disabled + details.put("noemail", "true"); + if (s.contains("nousertalk")) // cannot edit talk page + details.put("nousertalk", "true"); + details.put("expiry", s.substring(c, d)); + + // partial block parameters + if (s.contains("")) { - s.contains("anononly"), // anon-only - s.contains("nocreate"), // account creation blocked - s.contains("noautoblock"), // autoblock disabled - s.contains("noemail"), // email disabled - s.contains("nousertalk"), // cannot edit talk page - s.substring(c, d) // duration - }; + details.put("partial", "true"); + if (s.contains("")) + { + details.put("type", "namespaces"); + // TODO: add the actual namespaces + } + else if (s.contains("")) + { + details.put("type", "pages"); + // TODO: add the actual pages + } + } } } else if (type.equals(PROTECTION_LOG)) { - if (action.equals("unprotect")) - details = null; - else + if (action.equals("protect")) { // FIXME: return a protectionstate here? - int a = xml.indexOf("") + 7; - int b = xml.indexOf("", a); - details = xml.substring(a, b); + details.put("protection string", parseAttribute(xml, "description", 0)); } } else if (type.equals(USER_RENAME_LOG)) { int a = xml.indexOf("") + 7; int b = xml.indexOf("", a); - details = decode(xml.substring(a, b)); // the new username + details.put("new username", decode(xml.substring(a, b))); } else if (type.equals(USER_RIGHTS_LOG)) { - int a = xml.indexOf("new=\"") + 5; - int b = xml.indexOf('\"', a); - StringTokenizer tk = new StringTokenizer(xml.substring(a, b), ", "); - List temp = new ArrayList<>(); - while (tk.hasMoreTokens()) - temp.add(tk.nextToken()); - details = temp.toArray(new String[temp.size()]); + int a = xml.indexOf(""); + if (a != -1) { + int b = xml.indexOf("", a); + var oldgroups = xml.substring(a + 11, b); + var old_list = new ArrayList(); + for (int end = 0, start = oldgroups.indexOf(""); start != -1; start = oldgroups.indexOf("", end)) { + end = oldgroups.indexOf("", start); + old_list.add(oldgroups.substring(start + 3, end)); + } + details.put("oldgroups", String.join(",", old_list)); + } else // self-closing empty "" tag + details.put("oldgroups", ""); + int c = xml.indexOf(""); + if (c != -1) { + int d = xml.indexOf("", c); + var newgroups = xml.substring(c + 11, d); + var new_list = new ArrayList(); + for (int end = 0, start = newgroups.indexOf(""); start != -1; start = newgroups.indexOf("", end)) { + end = newgroups.indexOf("", start); + new_list.add(newgroups.substring(start + 3, end)); + } + details.put("newgroups", String.join(",", new_list)); + } else // self-closing empty "" tag + details.put("newgroups", ""); + } + + // tags + List tags = new ArrayList<>(); + if (xml.contains("")) + { + for (int idx = xml.indexOf(""); idx >= 0; idx = xml.indexOf("", ++idx)) + tags.add(xml.substring(idx + 5, xml.indexOf("", idx))); } - - LogEntry le = new LogEntry(type, action, reason, performer, target, timestamp, details); - le.userDeleted = userhidden; - le.reasonDeleted = reasonhidden; - le.targetDeleted = actionhidden; + + LogEntry le = new LogEntry(id, timestamp, user, reason, parsedreason, type, action, target, details); + le.setUserDeleted(userhidden); + le.setCommentDeleted(reasonhidden); + le.setContentDeleted(actionhidden); + le.setTags(tags); return le; } @@ -5424,7 +6085,7 @@ else if (type.equals(USER_RIGHTS_LOG)) * @throws IOException if a network error occurs * @since 0.15 */ - public String[] prefixIndex(String prefix) throws IOException + public List prefixIndex(String prefix) throws IOException { return listPages(prefix, null, ALL_NAMESPACES, -1, -1, null); } @@ -5437,7 +6098,7 @@ public String[] prefixIndex(String prefix) throws IOException * @throws IOException if a network error occurs * @since 0.15 */ - public String[] shortPages(int cutoff) throws IOException + public List shortPages(int cutoff) throws IOException { return listPages("", null, MAIN_NAMESPACE, -1, cutoff, null); } @@ -5451,7 +6112,7 @@ public String[] shortPages(int cutoff) throws IOException * @return pages below that size in that namespace * @since 0.15 */ - public String[] shortPages(int cutoff, int namespace) throws IOException + public List shortPages(int cutoff, int namespace) throws IOException { return listPages("", null, namespace, -1, cutoff, null); } @@ -5464,7 +6125,7 @@ public String[] shortPages(int cutoff, int namespace) throws IOException * @throws IOException if a network error occurs * @since 0.15 */ - public String[] longPages(int cutoff) throws IOException + public List longPages(int cutoff) throws IOException { return listPages("", null, MAIN_NAMESPACE, cutoff, -1, null); } @@ -5478,7 +6139,7 @@ public String[] longPages(int cutoff) throws IOException * @throws IOException if a network error occurs * @since 0.15 */ - public String[] longPages(int cutoff, int namespace) throws IOException + public List longPages(int cutoff, int namespace) throws IOException { return listPages("", null, namespace, cutoff, -1, null); } @@ -5500,7 +6161,7 @@ public String[] longPages(int cutoff, int namespace) throws IOException * @since 0.09 * @throws IOException if a network error occurs */ - public String[] listPages(String prefix, Map protectionstate, int namespace) throws IOException + public List listPages(String prefix, Map protectionstate, int namespace) throws IOException { return listPages(prefix, protectionstate, namespace, -1, -1, null); } @@ -5518,7 +6179,7 @@ public String[] listPages(String prefix, Map protectionstate, in * @param protectionstate a {@link #protect protection state}, use null * to not specify one * @param namespace a namespace. ALL_NAMESPACES is not suppported, an - * UnsupportedOperationException will be thrown. + * UnsupportedOperationException will be thrown unless a prefix is specified. * @param minimum the minimum size in bytes these pages can be. Use -1 to * not specify one. * @param maximum the maximum size in bytes these pages can be. Use -1 to @@ -5529,116 +6190,96 @@ public String[] listPages(String prefix, Map protectionstate, in * @since 0.09 * @throws IOException if a network error occurs */ - public String[] listPages(String prefix, Map protectionstate, int namespace, int minimum, + public List listPages(String prefix, Map protectionstate, int namespace, int minimum, int maximum, Boolean redirects) throws IOException { - // @revised 0.15 to add short/long pages // No varargs namespace here because MW API only supports one namespace // for this module. - StringBuilder url = new StringBuilder(query); - url.append("list=allpages"); - if (!prefix.isEmpty()) // prefix + Map getparams = new HashMap<>(); + getparams.put("list", "allpages"); + if (!prefix.isEmpty()) { - // cull the namespace prefix - namespace = namespace(prefix); - if (prefix.contains(":") && namespace != MAIN_NAMESPACE) - prefix = prefix.substring(prefix.indexOf(':') + 1); - url.append("&apprefix="); - url.append(encode(prefix, true)); + if (namespace == ALL_NAMESPACES) + { + namespace = namespace(prefix); + prefix = removeNamespace(prefix); + } + getparams.put("apprefix", normalize(prefix)); } else if (namespace == ALL_NAMESPACES) // check for namespace throw new UnsupportedOperationException("ALL_NAMESPACES not supported in MediaWiki API."); - url.append("&apnamespace="); - url.append(namespace); + getparams.put("apnamespace", String.valueOf(namespace)); if (protectionstate != null) { - StringBuilder apprtype = new StringBuilder("&apprtype="); - StringBuilder apprlevel = new StringBuilder("&apprlevel="); - for (Map.Entry entry : protectionstate.entrySet()) + StringBuilder apprtype = new StringBuilder(); + StringBuilder apprlevel = new StringBuilder(); + protectionstate.forEach((key, value) -> { - String key = entry.getKey(); if (key.equals("cascade")) - { - url.append("&apprfiltercascade="); - url.append((Boolean)entry.getValue() ? "cascading" : "noncascading"); - } + getparams.put("apprfiltercascade", (Boolean)value ? "cascading" : "noncascading"); else if (!key.contains("expiry")) { apprtype.append(key); - apprtype.append("%7C"); - apprlevel.append((String)entry.getValue()); - apprlevel.append("%7C"); + apprtype.append("|"); + apprlevel.append(value); + apprlevel.append("|"); } - } - apprtype.delete(apprtype.length() - 3, apprtype.length()); - apprlevel.delete(apprlevel.length() - 3, apprlevel.length()); - url.append(apprtype); - url.append(apprlevel); + }); + getparams.put("apprtype", apprtype.substring(0, apprtype.length() - 1)); + getparams.put("apprlevel", apprlevel.substring(0, apprlevel.length() - 1)); } // max and min - if (minimum != -1) - { - url.append("&apminsize="); - url.append(minimum); - } - if (maximum != -1) - { - url.append("&apmaxsize="); - url.append(maximum); - } + if (minimum >= 0) + getparams.put("apminsize", String.valueOf(minimum)); + if (maximum >= 0) + getparams.put("apmaxsize", String.valueOf(maximum)); if (redirects != null) - { - url.append("&apfilterredir="); - url.append(redirects ? "redirects" : "nonredirects"); - } + getparams.put("apfilterredir", redirects ? "redirects" : "nonredirects"); // set query limit = 1 request if max, min, prefix or protection level // not specified - int originallimit = getQueryLimit(); + int limit = -1; if (maximum < 0 && minimum < 0 && prefix.isEmpty() && protectionstate == null) - setQueryLimit(max); - List pages = queryAPIResult("ap", url, "listPages", (line, results) -> + limit = max; + List pages = makeListQuery("ap", getparams, null, "listPages", limit, (line, results) -> { // xml form:

for (int a = line.indexOf("

0; a = line.indexOf("

Warnings: + *

+ * + * @param page one of the qppage values specifed by the documentation below + * (case sensitive) * @return the list of pages returned by that particular special page * @throws IOException if a network error occurs - * @throws CredentialNotFoundException if page=Unwatchedpages and we cannot - * read it + * @throws SecurityException if the user lacks the privileges necessary to + * view a report (e.g. unwatchedpages) * @since 0.28 + * @see MediaWiki + * documentation */ - public String[] queryPage(String page) throws IOException, CredentialNotFoundException + public List queryPage(String page) throws IOException { - if (page.equals("Unwatchedpages") && (user == null || !user.isAllowedTo("unwatchedpages"))) - throw new CredentialNotFoundException("User does not have the \"unwatchedpages\" permission."); + Map getparams = new HashMap<>(); + getparams.put("list", "querypage"); + getparams.put("qppage", page); - StringBuilder url = new StringBuilder(query); - url.append("action=query&list=querypage&qppage="); - url.append(page); - - List pages = queryAPIResult("qp", url, "queryPage", (line, results) -> + List pages = makeListQuery("qp", getparams, null, "queryPage", -1, (line, results) -> { // xml form: for (int x = line.indexOf(" 0; x = line.indexOf("amount most recently created pages in the main - * namespace. WARNING: The - * recentchanges table stores new pages for a finite period of time; - * it is not possible to retrieve pages created before then. + * Fetches recently created pages. See {@link #recentChanges(Wiki.RequestHelper, + * String)} for full documentation. Equivalent to [[Special:Newpages]]. * - * @param amount the number of pages to fetch (overrides global query - * limits) - * @return the revisions that created the pages satisfying the requirements - * above - * @throws IOException if a network error occurs - * @since 0.20 - */ - public Revision[] newPages(int amount) throws IOException - { - return recentChanges(amount, 0, true, MAIN_NAMESPACE); - } - - /** - * Fetches the amount most recently created pages in the main - * namespace subject to the specified constraints. WARNING: The recentchanges - * table stores new pages for a finite period of - * time; it is not possible to retrieve pages created before then. - * Equivalent to [[Special:Newpages]]. - * - * @param rcoptions a bitmask of {@link #HIDE_ANON} etc that dictate which - * pages we return (e.g. to exclude patrolled pages set rcoptions = HIDE_PATROLLED). - * @param amount the amount of new pages to get (overrides global query - * limits) - * @param ns a list of namespaces to filter by, empty = all namespaces. + * @param helper a {@link Wiki.RequestHelper} (optional, use null to not + * provide any optional parameters * @return the revisions that created the pages satisfying the requirements * above * @throws IOException if a network error occurs - * @since 0.20 + * @since 0.35 */ - public Revision[] newPages(int amount, int rcoptions, int... ns) throws IOException + public List newPages(Wiki.RequestHelper helper) throws IOException { - return recentChanges(amount, rcoptions, true, ns); + return recentChanges(helper, "new"); } /** - * Fetches the amount most recent changes in the main namespace. - * WARNING: The - * recentchanges table stores new pages for a finite period of - * time; it is not possible to retrieve pages created before then. + * Fetches recent edits to this wiki. See {@link + * #recentChanges(Wiki.RequestHelper, String)} for full documentation. * Equivalent to [[Special:Recentchanges]]. - *

- * Note: Log entries in recent changes have a revid of 0! * - * @param amount the number of entries to return (overrides global query - * limits) + * @param helper a {@link Wiki.RequestHelper} (optional, use null to not + * provide any optional parameters * @return the recent changes that satisfy these criteria * @throws IOException if a network error occurs * @since 0.23 */ - public Revision[] recentChanges(int amount) throws IOException + public List recentChanges(Wiki.RequestHelper helper) throws IOException { - return recentChanges(amount, 0, false, MAIN_NAMESPACE); + return recentChanges(helper, null); } /** - * Fetches the amount most recent changes in the specified - * namespace. WARNING: The recent changes table only stores new pages for - * about a month. It is not possible to retrieve changes before then. - * Equivalent to [[Special:Recentchanges]]. + * Fetches recent changes to this wiki. WARNING: The recentchanges + * table stores edits for a + * finite period of time; it is not possible to retrieve pages created + * before then. Equivalent to [[Special:Recentchanges]]. + * *

- * Note: Log entries in recent changes have a revid of 0! + * Accepted parameters from helper are: + *

    + *
  • {@link Wiki.RequestHelper#withinDateRange(OffsetDateTime, + * OffsetDateTime) date range} + *
  • {@link Wiki.RequestHelper#byUser(String) user} + *
  • {@link Wiki.RequestHelper#notByUser(String) not by user} + *
  • {@link Wiki.RequestHelper#reverse(boolean) reverse} + *
  • {@link Wiki.RequestHelper#inNamespaces(int...) namespaces} + *
  • {@link Wiki.RequestHelper#taggedWith(String) tag} + *
  • {@link Wiki.RequestHelper#filterBy(Map) filter by}: "minor", "bot", + * "anon", "redirect", "patrolled" + *
  • {@link Wiki.RequestHelper#limitedTo(int) local query limit} + *
* - * @param amount the number of entries to return (overrides global query - * limits) - * @param ns a list of namespaces to filter by, empty = all namespaces. - * @return the recent changes that satisfy these criteria - * @throws IOException if a network error occurs - * @since 0.23 - */ - public Revision[] recentChanges(int amount, int[] ns) throws IOException - { - return recentChanges(amount, 0, false, ns); - } - - /** - * Fetches the amount most recent changes in the specified - * namespace subject to the specified constraints. WARNING: The recent - * changes table only stores new pages for about a month. It is not - * possible to retrieve changes before then. Equivalent to - * [[Special:Recentchanges]]. *

- * Note: Log entries in recent changes have a revid of 0! + * If {@code rctype} is not {@code "edit"} or {@code "new"} then the results + * consist of pseudo-revisions whose data does not correspond to an actual + * on-wiki state. For example: * - * @param amount the number of entries to return (overrides global query - * limits) - * @param ns a list of namespaces to filter by, empty = all namespaces. - * @param rcoptions a bitmask of HIDE_ANON etc that dictate which pages - * we return. + *

    + *
  • {@code rctype == "log"} yields {@code id == 0} and {@code title} is + * the log entry target + *
  • {@code rctype == "external"} yields {@code id} as the most recent + * edit to {@code title}, {@code previous_id == id}, {@code user} is + * the external user making the change, {@code sizediff == 0} and + * {@code comment} describes the external change + *
  • {@code rctype =="categorize"} yields {@code title} as the category + * added or removed and {@code comment} specifies the page added or + * removed to that category + *
+ * + * @param helper a {@link Wiki.RequestHelper} (optional, use null to not + * provide any of the optional parameters described above + * @param rctype null, "edit" (edits only) or "new" (new pages); your + * mileage may vary for other types (log, external, categorize) * @return the recent changes that satisfy these criteria * @throws IOException if a network error occurs - * @since 0.23 + * @since 0.35 */ - public Revision[] recentChanges(int amount, int rcoptions, int... ns) throws IOException + protected List recentChanges(Wiki.RequestHelper helper, String rctype) throws IOException { - return recentChanges(amount, rcoptions, false, ns); - } + int limit = -1; + Map getparams = new HashMap<>(); + getparams.put("list", "recentchanges"); + getparams.put("rcprop", "title|ids|user|timestamp|flags|comment|parsedcomment|sizes|sha1|tags"); + if (helper != null) + { + helper.setRequestType("rc"); + getparams.putAll(helper.addNamespaceParameter()); + getparams.putAll(helper.addUserParameter()); + getparams.putAll(helper.addExcludeUserParameter()); + getparams.putAll(helper.addDateRangeParameters()); + getparams.putAll(helper.addTagParameter()); + getparams.putAll(helper.addReverseParameter()); + getparams.putAll(helper.addShowParameter()); + limit = helper.limit(); + } - /** - * Fetches the amount most recent changes in the specified - * namespace subject to the specified constraints. WARNING: The recent - * changes table only stores new pages for about a month. It is not - * possible to retrieve changes before then. Equivalent to - * [[Special:Recentchanges]]. - *

- * Note: Log entries in recent changes have a revid of 0! - * - * @param amount the number of entries to return (overrides global - * query limits) - * @param ns a list of namespaces to filter by, empty = all namespaces. - * @param rcoptions a bitmask of HIDE_ANON etc that dictate which pages - * we return. - * @param newpages show new pages only - * @return the recent changes that satisfy these criteria - * @throws IOException if a network error occurs - * @since 0.23 - */ - protected Revision[] recentChanges(int amount, int rcoptions, boolean newpages, int... ns) throws IOException - { - StringBuilder url = new StringBuilder(query); - url.append("list=recentchanges&rcprop=title%7Cids%7Cuser%7Ctimestamp%7Cflags%7Ccomment%7Csizes%7Csha1"); - constructNamespaceString(url, "rc", ns); - if (newpages) - url.append("&rctype=new"); - // rc options - if (rcoptions > 0) - { - url.append("&rcshow="); - if ((rcoptions & HIDE_ANON) == HIDE_ANON) - url.append("!anon%7C"); - if ((rcoptions & HIDE_SELF) == HIDE_SELF) - url.append("!self%7C"); - if ((rcoptions & HIDE_MINOR) == HIDE_MINOR) - url.append("!minor%7C"); - if ((rcoptions & HIDE_PATROLLED) == HIDE_PATROLLED) - url.append("!patrolled%7C"); - if ((rcoptions & HIDE_BOT) == HIDE_BOT) - url.append("!bot%7C"); - // chop off last | - url.delete(url.length() - 3, url.length()); - } - - int originallimit = getQueryLimit(); - setQueryLimit(amount); - List revisions = queryAPIResult("rc", url, newpages ? "newPages" : "recentChanges", + if (rctype != null) + getparams.put("rctype", rctype); + + List revisions = makeListQuery("rc", getparams, null, "recentChanges", limit, (line, results) -> { // xml form for (int i = line.indexOf(" 0; i = line.indexOf("", i); + int j = line.indexOf("", i); results.add(parseRevision(line.substring(i, j), "")); } }); - setQueryLimit(originallimit); - int temp = revisions.size(); - log(Level.INFO, "recentChanges", "Successfully retrieved recent changes (" + temp + " revisions)"); - return revisions.toArray(new Revision[temp]); + log(Level.INFO, "recentChanges", "Successfully retrieved recent changes (" + revisions.size() + " revisions)"); + return revisions; } /** * Fetches all pages that use interwiki links to the specified wiki and the - * page on that wiki that is linked to. For example, - * getInterWikiBacklinks("testwiki") may return: + * page on that wiki that is linked to. For example, {@code + * getInterWikiBacklinks("testwiki")} may return: * * { * { "Spam", "testwiki:Blah" }, @@ -5844,7 +6431,7 @@ protected Revision[] recentChanges(int amount, int rcoptions, boolean newpages, * @throws IOException if a network error occurs * @since 0.23 */ - public String[][] getInterWikiBacklinks(String prefix) throws IOException + public List getInterWikiBacklinks(String prefix) throws IOException { return getInterWikiBacklinks(prefix, "|"); } @@ -5858,8 +6445,8 @@ public String[][] getInterWikiBacklinks(String prefix) throws IOException * *

* Example: If [[Test]] and [[Spam]] both contain the interwiki link - * [[testwiki:Blah]] then getInterWikiBacklinks("testwiki", "Blah"); - * will return (sorted by title) + * [[testwiki:Blah]] then {@code getInterWikiBacklinks("testwiki", "Blah");} + * will return (sorted by title) * * { * { "Spam", "testwiki:Blah" }, @@ -5882,23 +6469,20 @@ public String[][] getInterWikiBacklinks(String prefix) throws IOException * prefix (the MediaWiki API doesn't like this) * @since 0.23 */ - public String[][] getInterWikiBacklinks(String prefix, String title) throws IOException + public List getInterWikiBacklinks(String prefix, String title) throws IOException { // must specify a prefix if (title.equals("|") && prefix.isEmpty()) throw new IllegalArgumentException("Interwiki backlinks: title specified without prefix!"); - StringBuilder url = new StringBuilder(query); - url.append("list=iwbacklinks&iwblprefix="); - url.append(prefix); + Map getparams = new HashMap<>(); + getparams.put("list", "iwbacklinks"); + getparams.put("iwblprefix", prefix); if (!title.equals("|")) - { - url.append("&iwbltitle="); - url.append(title); - } - url.append("&iwblprop=iwtitle%7Ciwprefix"); + getparams.put("iwbltitle", normalize(title)); + getparams.put("iwblprop", "iwtitle|iwprefix"); - List links = queryAPIResult("iwbl", url, "getInterWikiBacklinks", (line, results) -> + List links = makeListQuery("iwbl", getparams, null, "getInterWikiBacklinks", -1, (line, results) -> { // xml form: for (int x = line.indexOf(" 0; x = line.indexOf(" { - private String username; - private String[] rights = null; // cache - private String[] groups = null; // cache + private final String username; + private final OffsetDateTime registration; + // user privileges (volatile, changes rarely) + private List rights; + private List groups; + private boolean blocked; + // user preferences (volatile, changes rarely) + private Gender gender; + private boolean emailable; + // volatile, changes often + private int editcount; /** * Creates a new user object. Does not create a new user on the @@ -5933,11 +6525,27 @@ public class User implements Cloneable, Serializable * be called for anons. * * @param username the username of the user + * @param registration when the user was registered + * @param rights the rights this user has + * @param groups the groups this user belongs to + * @param gender the self-declared {@link Wiki.Gender Gender} of this user. + * @param emailable whether the user can be emailed through [[Special:Emailuser]] + * @param blocked whether this user is blocked + * @param editcount the internal edit count of this user * @since 0.05 */ - protected User(String username) + protected User(String username, OffsetDateTime registration, List rights, List groups, + Gender gender, boolean emailable, boolean blocked,int editcount) { - this.username = username; + this.username = Objects.requireNonNull(username); + // can be null per https://phabricator.wikimedia.org/T24097 + this.registration = registration; + this.rights = Objects.requireNonNull(rights); + this.groups = Objects.requireNonNull(groups); + this.gender = gender; + this.emailable = emailable; + this.blocked = blocked; + this.editcount = editcount; } /** @@ -5945,111 +6553,131 @@ protected User(String username) * @return this user's username * @since 0.08 */ - public String getUsername() + public final String getUsername() { return username; } /** - * Gets various properties of this user. Returns: - *

    - *
  • editcount: (int) {@link #countEdits()} the user's edit - * count - *
  • groups: (String[]) the groups the user is in (see - * [[Special:Listgrouprights]]) - *
  • rights: (String[]) the stuff the user can do - *
  • emailable: (Boolean) whether the user can be emailed - * through [[Special:Emailuser]] or emailUser() - *
  • blocked: (Boolean) whether the user is blocked - *
  • gender: (Wiki.Gender) the user's gender - *
  • created: (Calendar) when the user account was created - *
- * + * Gets the date/time at which this user account was created. May be + * {@code null} per + * https://phabricator.wikimedia.org/T24097. * @return (see above) - * @throws IOException if a network error occurs - * @since 0.24 + * @since 0.35 */ - public Map getUserInfo() throws IOException + public final OffsetDateTime getRegistrationDate() { - return Wiki.this.getUserInfo(new String[] { username })[0]; + return registration; } /** - * Returns true if the user is allowed to perform the specified action. - * Uses the rights cache. Read [[Special:Listgrouprights]] before using - * this! + * Returns {@code true} if the user is allowed to perform the specified + * action(s). Read [[Special:Listgrouprights]] before using this! * @param right a specific action - * @return whether the user is allowed to execute it + * @param morerights additional actions to check + * @return whether the user is allowed to execute them * @since 0.24 - * @throws IOException if a network error occurs */ - public boolean isAllowedTo(String right) throws IOException - { - // We can safely assume the user is allowed to { read, edit, create, - // writeapi }. - if (rights == null) - rights = (String[])getUserInfo().get("rights"); - for (String r : rights) - if (r.equals(right)) - return true; - return false; + public boolean isAllowedTo(String right, String... morerights) + { + List temp = new ArrayList<>(); + temp.add(right); + temp.addAll(List.of(morerights)); + return rights.containsAll(temp); } /** - * Returns true if the user is a member of the specified group. Uses - * the groups cache. + * Returns {@code true} if the user is a member of the specified group. * @param group a specific group * @return whether the user is in it * @since 0.24 - * @throws IOException if a network error occurs */ - public boolean isA(String group) throws IOException + public boolean isA(String group) { - if (groups == null) - groups = (String[])getUserInfo().get("groups"); - for (String g : groups) - if (g.equals(group)) - return true; - return false; + return groups.contains(group); } /** - * Returns a log of the times when the user has been blocked. - * @return records of the occasions when this user has been blocked - * @throws IOException if something goes wrong - * @since 0.08 + * Returns the groups the user is a member of. See [[Special:Listgrouprights]]. + * Changes in this list do not propagate to this object or the wiki. + * @return (see above) + * @since 0.35 + */ + public List getGroups() + { + return new ArrayList<>(groups); + } + + /** + * Returns the specific permissions this user has. See [[Special:Listgrouprights]]. + * Changes in this list do not propagate to the object or the wiki. + * @return (see above) + * @since 0.35 + */ + public List getRights() + { + return new ArrayList<>(rights); + } + + /** + * Returns whether this user can be emailed through [[Special:Emailuser]]. + * @return (see above) + * @see #emailUser(Wiki.User, String, String, boolean) + * @since 0.35 + */ + public boolean canBeEmailed() + { + return emailable; + } + + /** + * Returns the self-disclosed {@linkplain Wiki.Gender gender} of this + * user. + * @return (see above) + * @see Wiki.Gender + * @since 0.35 */ - public LogEntry[] blockLog() throws IOException + public Gender getGender() { - return Wiki.this.getLogEntries(Wiki.BLOCK_LOG, null, null, "User:" + username); + return gender; } /** - * Determines whether this user is blocked by looking it up on the IP - * block list. + * Determines whether this user is blocked at the time of construction. + * If you want a live check, look up the user on the {@linkplain + * #getBlockList list of blocks}. * @return whether this user is blocked - * @throws IOException if we cannot retrieve the IP block list * @since 0.12 */ - public boolean isBlocked() throws IOException + public boolean isBlocked() { - // @revised 0.18 now check for errors after each edit, including blocks - return getIPBlockList(username, null, null).length != 0; + return blocked; } /** - * Fetches the internal edit count for this user, which includes all - * live edits and deleted edits after (I think) January 2007. If you - * want to count live edits only, use the slower - * int count = {@link User#contribs(int...) user.contribs()}.length;. + * Fetches the internal edit count for this user at the time of + * construction, which includes all live edits and deleted edits after + * (I think) January 2007. If you want to count live edits only, + * compute the size of {@link User#contribs(int...) User.contribs()}. * * @return the user's edit count - * @throws IOException if a network error occurs * @since 0.16 */ - public int countEdits() throws IOException + public int countEdits() { - return (Integer)getUserInfo().get("editcount"); + return editcount; + } + + /** + * Returns a log of the times when the user has been blocked. + * @return records of the occasions when this user has been blocked + * @throws IOException if something goes wrong + * @since 0.08 + */ + public List blockLog() throws IOException + { + Wiki.RequestHelper rh = new RequestHelper().byTitle("User:" + username); + return Wiki.this.getLogEntries(Wiki.BLOCK_LOG, null, rh); } /** @@ -6059,243 +6687,430 @@ public int countEdits() throws IOException * @throws IOException if a network error occurs * @since 0.17 */ - public Revision[] contribs(int... ns) throws IOException + public List contribs(int... ns) throws IOException { - return Wiki.this.contribs(username, ns); + Wiki.RequestHelper rh = new RequestHelper().inNamespaces(ns); + return Wiki.this.contribs(username, rh); } - + /** * Returns the list of logged actions performed by this user. * @param logtype what log to get ({@link Wiki#DELETION_LOG} etc.) - * @param action what action to get (e.g. delete, undelete), use + * @param action what action to get (e.g. delete, undelete), use * "" to not specify one * @return (see above) * @throws IOException if a network error occurs * @since 0.33 */ - public LogEntry[] getLogEntries(String logtype, String action) throws IOException + public List getLogEntries(String logtype, String action) throws IOException { - return Wiki.this.getLogEntries(logtype, action, username, null); + Wiki.RequestHelper rh = new RequestHelper().byUser(username); + return Wiki.this.getLogEntries(logtype, action, rh); } /** - * Copies this user object. - * @return the copy - * @throws CloneNotSupportedException if the clone fails - * @since 0.08 + * Tests whether this user is equal to another one. + * @param x another object + * @return whether the usernames of the users are equal */ @Override - public User clone() throws CloneNotSupportedException + public boolean equals(Object x) { - try - { - return (User)super.clone(); - } - catch (CloneNotSupportedException e) - { - return null; - } + if (!(x instanceof User)) + return false; + User other = (User)x; + return Objects.equals(username, other.username) + && Objects.equals(registration, other.registration); } /** - * Tests whether this user is equal to another one. - * @param x another object - * @return whether the usernames of the users are equal - * @since 0.08 + * Returns a hashcode of this user based on the username and + * registration date. + * @return see above */ @Override - public boolean equals(Object x) + public int hashCode() { - return x instanceof User && username.equals(((User)x).username); + return username.hashCode() * 127 + registration.hashCode(); } /** - * Returns a string representation of this user. - * @return see above - * @since 0.17 + * Enables sorting of users by their username. + * @param other some other user + * @return less than zero if this user is alphabetically before the + * other, 0 if they are the same and 1 if alphabetically after */ @Override - public String toString() + public int compareTo(User other) { - StringBuilder temp = new StringBuilder("User[username="); - temp.append(username); - temp.append("groups="); - temp.append(groups != null ? Arrays.toString(groups) : "unset"); - temp.append("]"); - return temp.toString(); + return username.compareTo(other.username); } /** - * Returns a hashcode of this user. + * Returns a string representation of this user. * @return see above - * @since 0.19 */ @Override - public int hashCode() + public String toString() { - return username.hashCode() * 2 + 1; + return getClass().getName() + + "[username=" + username + + ",registration=" + (registration != null ? registration.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) : "unset") + + ",groups=" + Arrays.toString(groups.toArray()) + "]"; } } /** - * A wrapper class for an entry in a wiki log, which represents an action - * performed on the wiki. - * - * @see #getLogEntries - * @since 0.08 + * A data super class for an event happening on a wiki such as a {@link + * Wiki.Revision} or a {@link Wiki.LogEntry}. + * @since 0.35 */ - public class LogEntry implements Comparable + public abstract class Event implements Comparable { - // internal data storage - private long logid = -1; - private String type; - private String action; - private final String reason; - private User user; - private String target; + private final long id; private final OffsetDateTime timestamp; - private Object details; - private boolean reasonDeleted = false, userDeleted = false, - targetDeleted = false; + private final String user; + private final String title; + private final String comment; + private final String parsedcomment; + private List tags; + private boolean commentDeleted = false, userDeleted = false, + contentDeleted = false; /** - * Creates a new log entry. WARNING: does not perform the action - * implied. Use Wiki.class methods to achieve this. - * - * @param type the type of log entry, one of {@link #USER_CREATION_LOG}, - * {@link #DELETION_LOG}, {@link #BLOCK_LOG}, etc. - * @param action the type of action that was performed e.g. "delete", - * "unblock", "overwrite", etc. - * @param reason why the action was performed - * @param user the user who performed the action - * @param target the target of the action - * @param timestamp the local time when the action was performed. - * We will convert this back into an OffsetDateTime - * @param details the details of the action (e.g. the new title of - * the page after a move was performed). - * @since 0.08 + * Creates a new Event record. + * @param id the unique ID of the event + * @param timestamp the timestamp at which it occurred + * @param user the user or IP address performing the event + * @param title the title of the page affected + * @param comment the comment left by the user when performing the + * event (e.g. an edit summary) + * @param parsedcomment comment, but parsed into HTML */ - protected LogEntry(String type, String action, String reason, User user, - String target, OffsetDateTime timestamp, Object details) + protected Event(long id, OffsetDateTime timestamp, String user, String title, String comment, String parsedcomment) { - this.type = type; - this.action = action; - this.reason = reason; + this.id = id; + this.timestamp = Objects.requireNonNull(timestamp); this.user = user; - this.target = target; - this.timestamp = timestamp; - this.details = details; + this.title = title; + this.comment = comment; + // Rewrite parsedcomments to fix useless relative hyperlinks to + // other wiki pages + if (parsedcomment == null) + this.parsedcomment = null; + else + this.parsedcomment = parsedcomment.replace("href=\"/wiki", "href=\"" + protocol + domain + "/wiki"); } - + + /** + * Gets the unique ID of this event. For a {@link Wiki.Revision}, this + * number is referred to as the "oldid" or "revid" and should not be + * confused with "rcid" (which is the ID in the recentchanges table). + * For a {@link Wiki.LogEntry}, this value only makes sense if the + * record was obtained through {@link Wiki#getLogEntries(String, String, + * RequestHelper)} and overloads (other methods return pseudo-LogEntries). + * @return the ID of this revision + */ + public long getID() + { + return id; + } + + /** + * Gets the timestamp of this event. + * @return the timestamp of this event + */ + public OffsetDateTime getTimestamp() + { + return timestamp; + } + /** - * Gets the ID of this log entry. Only available if retrieved by - * {@link Wiki#getLogEntries}, otherwise returns -1. + * Returns the user or anon who performed this event. You should pass + * this (if not an IP) to {@link #getUser} to obtain a {@link + * Wiki.User} object. Returns {@code null} if the user was + * RevisionDeleted and you lack the necessary privileges. + * @return the user or anon + */ + public String getUser() + { + return user; + } + + /** + * Sets a boolean flag that the user triggering this event has been + * RevisionDeleted in on-wiki records. + * @param deleted (see above) + * @see #isUserDeleted() + * @see #getUser() + */ + protected void setUserDeleted(boolean deleted) + { + userDeleted = deleted; + } + + /** + * Returns {@code true} if the user triggering this event is + * RevisionDeleted. * @return (see above) - * @since 0.33 + * @see #getUser() */ - public long getLogID() + public boolean isUserDeleted() { - return logid; + return userDeleted; } /** - * Gets the type of log that this entry is in. - * @return one of {@link Wiki#DELETION_LOG}, {@link Wiki#BLOCK_LOG}, etc. - * @since 0.08 + * Returns the page affected by this event. May be {@code null} for + * certain types of LogEntry and/or if the LogEntry is RevisionDeleted + * and you don't have the ability to access it. + * @return (see above) + * @see #isContentDeleted() */ - public String getType() + public String getTitle() { - return type; + return title; } /** - * Gets a string description of the action performed, for example - * "delete", "protect", "overwrite", ... WARNING: returns null if the - * action was RevisionDeleted. - * @return the type of action performed - * @since 0.08 + * Gets the comment for this event in wikitext. If this is a {@link + * Wiki.Revision}, this is the edit summary. If this is a {@link + * Wiki.LogEntry}, this is the reason for the logged action. WARNING: + * returns {@code null} if the reason was RevisionDeleted and you lack + * the necessary privileges. + * @return the comment associated with the event + * @see #getParsedComment() */ - public String getAction() + public String getComment() { - return action; + return comment; } - + + /** + * Gets the comment for this event, with limited parsing into HTML. + * Hyperlinks in the returned HTML are rewritten from useless relative + * URLs to full URLs that point to the wiki page in question. Returns + * {@code null} if {@linkplain #isCommentDeleted() the comment was + * RevisionDeleted} and you lack the necessary privileges. + * + *

Warnings: + *

    + *
  • Not available through {@link #getBlockList}. + *
+ * + * @return the comment associated with the event, parsed into HTML + * @see #getComment() + */ + public String getParsedComment() + { + return parsedcomment; + } + /** - * Returns true if the target has been RevisionDeleted (action is hidden - * in the GUI but retrievable by the API). + * Sets a boolean flag that the comment associated with this event has + * been RevisionDeleted in on-wiki records. + * @param deleted (see above) + * @see #getComment + * @see #getParsedComment() + * @see #isCommentDeleted() + */ + protected void setCommentDeleted(boolean deleted) + { + commentDeleted = deleted; + } + + /** + * Returns {@code true} if the comment is RevisionDeleted. * @return (see above) - * @since 0.32 + * @see #getComment + * @see #getParsedComment() */ - public boolean isTargetDeleted() + public boolean isCommentDeleted() { - return targetDeleted; + return commentDeleted; } /** - * Gets the reason supplied by the perfoming user when the action - * was performed. WARNING: returns null if the reason was - * RevisionDeleted and one does not have access to the content. - * @return the reason the action was performed - * @since 0.08 + * Sets a boolean flag that the content of this event has been + * RevisionDeleted. + * @param deleted (see above) + * @see #isContentDeleted() + */ + protected void setContentDeleted(boolean deleted) + { + contentDeleted = deleted; + } + + /** + * Returns {@code true} if the content of this event has been + * RevisionDeleted. For a {@link Wiki.LogEntry}, this refers to the + * page the logged action affects and the logged action performed (e.g. + * "unblock" or "delete"). + * @return (see above) */ - public String getReason() + public boolean isContentDeleted() { - return reason; + return contentDeleted; } /** - * Returns true if the reason is RevisionDeleted. + * Returns the list of tags attached to this event. Modifying the + * return value does not affect this Revision object or the wiki state. * @return (see above) - * @since 0.32 + * @see MediaWiki + * documentation + * @since 0.37 */ - public boolean isReasonDeleted() + public List getTags() { - return reasonDeleted; + return new ArrayList<>(tags); + } + + /** + * Sets the list of tags attached to this event. Modifying the supplied + * list does not affect this Revision object or change on-wiki state. + * @param tags a list of change tags + * @see MediaWiki + * documentation + * @since 0.37 + */ + protected void setTags(List tags) + { + this.tags = new ArrayList<>(tags); } /** - * Gets the user object representing who performed the action. - * WARNING: returns null if the user was RevisionDeleted and one does - * not have access to the content. - * @return the user who performed the action. - * @since 0.08 + * Returns a String representation of this Event. Subclasses only need + * to lop off the trailing "]" and add their own fields when overriding + * this method. + * @return (see above) + */ + @Override + public String toString() + { + return getClass().getName() + + "[id=" + id + + ",timestamp=" + timestamp.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + + ",user=\"" + Objects.toString(user, "[DELETED]") + "\"" + + ",userDeleted=" + userDeleted + + ",title=\"" + Objects.toString(title, "[null or deleted]") + "\"" + + ",comment=\"" + Objects.toString(comment, "[DELETED]") + "\"" + + ",commentDeleted=" + commentDeleted + + ",contentDeleted=" + contentDeleted + ']'; + } + + /** + * Determines whether this Event is equal to some other object. This + * method checks the ID, timestamp, user, title and comment. + * @param other the other object to compare to + * @return whether this instance is equal to that object */ - public User getUser() + @Override + public boolean equals(Object other) { - return user; + if (!(other instanceof Event)) + return false; + Event event = (Event)other; + return id == event.id + && Objects.equals(timestamp, event.timestamp) + && Objects.equals(user, event.user) + && Objects.equals(title, event.title) + && Objects.equals(comment, event.comment); } - + /** - * Returns true if the user who performed this LogEntry is - * RevisionDeleted. + * Returns a hash code for this object based on the ID, timestamp, + * user, title and comment. * @return (see above) - * @since 0.32 */ - public boolean isUserDeleted() + @Override + public int hashCode() { - return userDeleted; + int hc = Long.hashCode(id); + hc = 127 * hc + timestamp.hashCode(); + hc = 127 * hc + Objects.hashCode(user); + hc = 127 * hc + Objects.hashCode(title); + hc = 127 * hc + Objects.hashCode(comment); + return hc; + } + + /** + * Compares this event to another one based on the recentness of their + * timestamps (more recent = positive return value), then + * alphabetically by user. + * @param other the event to compare to + * @return the comparator value, negative if less, positive if greater + */ + @Override + public int compareTo(Wiki.Event other) + { + int result = timestamp.compareTo(other.timestamp); + if (result == 0 && user != null) + result = user.compareTo(other.user); + return result; + } + } + + /** + * A wrapper class for an entry in a wiki log, which represents an action + * performed on the wiki. + * + * @see #getLogEntries + * @since 0.08 + */ + public class LogEntry extends Event + { + private final String type; + private String action; + private Map details; + + /** + * Creates a new log entry. WARNING: does not perform the action + * implied. Use Wiki.class methods to achieve this. + * + * @param id the unique of this log entry + * @param timestamp the local time when the action was performed. + * @param user the user who performed the action + * @param comment why the action was performed + * @param parsedcomment like comment, but parsed into HTML + * @param type the type of log entry, one of {@link #USER_CREATION_LOG}, + * {@link #DELETION_LOG}, {@link #BLOCK_LOG}, etc. + * @param action the type of action that was performed e.g. "delete", + * "unblock", "overwrite", etc. + * @param target the target of the action + * @param details the details of the action (e.g. the new title of + * the page after a move was performed). + * @since 0.08 + */ + protected LogEntry(long id, OffsetDateTime timestamp, String user, String comment, + String parsedcomment, String type, String action, String target, Map details) + { + super(id, timestamp, user, target, comment, parsedcomment); + this.type = Objects.requireNonNull(type); + this.action = action; + this.details = details; } /** - * Gets the target of the action represented by this log entry. WARNING: - * returns null if the content was RevisionDeleted and one does not - * have access to the content. - * @return the target of this log entry + * Gets the type of log that this entry is in. + * @return one of {@link Wiki#DELETION_LOG}, {@link Wiki#BLOCK_LOG}, etc. * @since 0.08 */ - public String getTarget() + public String getType() { - return target; + return type; } /** - * Gets the timestamp of this log entry. - * @return the timestamp of this log entry + * Gets a string description of the action performed, for example + * "delete", "protect", "overwrite", ... WARNING: returns null if the + * action was RevisionDeleted. + * @return the type of action performed * @since 0.08 */ - public OffsetDateTime getTimestamp() + public String getAction() { - return timestamp; + return action; } /** @@ -6309,13 +7124,14 @@ public OffsetDateTime getTimestamp() * USER_RENAME_LOG * The new username * BLOCK_LOG - * new Object[] { boolean anononly, boolean nocreate, boolean + * new Object[] { boolean anononly, boolean nocreate, boolean * noautoblock, boolean noemail, boolean nousertalk, String duration } * USER_RIGHTS_LOG - * The new user rights (String[]) + * The old ("oldgroups") and new ("newgroups") user rights, + * comma-separated * PROTECTION_LOG * if action == "protect" or "modify" return the protection level - * (int, -2 if unrecognized) if action == "move_prot" return + * (int, -2 if unrecognized) if action == "move_prot" return * the old title, else null * Others or RevisionDeleted * null @@ -6329,7 +7145,7 @@ public OffsetDateTime getTimestamp() * @return the details of the log entry * @since 0.08 */ - public Object getDetails() + public Map getDetails() { return details; } @@ -6342,44 +7158,21 @@ public Object getDetails() @Override public String toString() { - StringBuilder s = new StringBuilder("LogEntry[logid="); - s.append(logid); + StringBuilder s = new StringBuilder(super.toString()); + s.deleteCharAt(s.length() - 1); s.append(",type="); s.append(type); s.append(",action="); - s.append(action == null ? "[hidden]" : action); - s.append(",user="); - s.append(user == null ? "[hidden]" : user.getUsername()); - s.append(",timestamp="); - s.append(timestamp.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - s.append(",target="); - s.append(target == null ? "[hidden]" : target); - s.append(",reason=\""); - s.append(reason == null ? "[hidden]" : reason); - s.append("\",details="); - if (details instanceof Object[]) - s.append(Arrays.asList((Object[])details)); // crude formatting hack - else - s.append(details); + s.append(Objects.toString(action, "[DELETED]")); + s.append(",details="); + s.append(details); s.append("]"); return s.toString(); } /** - * Compares this log entry to another one based on the recentness - * of their timestamps. - * @param other the log entry to compare - * @return whether this object is equal to - * @since 0.18 - */ - @Override - public int compareTo(Wiki.LogEntry other) - { - return timestamp.compareTo(other.timestamp); - } - - /** - * Determines whether two LogEntries refer to the same event. + * Determines whether two LogEntries are equal based on the underlying + * {@linkplain Event#equals(Object) Event}, type and action. * @param other some object to compare to * @return (see above) * @since 0.33 @@ -6387,12 +7180,28 @@ public int compareTo(Wiki.LogEntry other) @Override public boolean equals(Object other) { + if (!super.equals(other)) + return false; if (!(other instanceof LogEntry)) return false; LogEntry le = (LogEntry)other; - return type.equals(le.type) && action.equals(le.action) && - user.equals(le.user) && target.equals(le.target) && - reason.equals(le.reason) && timestamp.equals(le.timestamp); + return Objects.equals(type, le.type) + && Objects.equals(action, le.action); + } + + /** + * Computes a hashcode for this LogEntry based on the underlying + * {@linkplain Event#hashCode() Event}, type and action. + * @return (see above) + * @since 0.35 + */ + @Override + public int hashCode() + { + int hc = super.hashCode(); + hc = 127 * hc + type.hashCode(); + hc = 127 * hc + Objects.hashCode(action); + return hc; } } @@ -6400,46 +7209,39 @@ public boolean equals(Object other) * Represents a contribution and/or a revision to a page. * @since 0.17 */ - public class Revision implements Comparable + public class Revision extends Event { - private boolean minor, bot, rvnew; - private String summary; - private long revid, rcid = -1; + private final boolean minor, bot, rvnew; + private final String sha1; + private long rcid = -1; private long previous = 0, next = 0; - private OffsetDateTime timestamp; - private String user; - private String title; - private String rollbacktoken = null; - private int size = 0; - private int sizediff = 0; - private boolean summaryDeleted = false, userDeleted = false, contentDeleted = false; + private int size = 0, sizediff = 0; private boolean pageDeleted = false; /** * Constructs a new Revision object. * @param revid the id of the revision (this is a long since - * {{NUMBEROFEDITS}} on en.wikipedia.org is now (January 2012) ~25% - * of Integer.MAX_VALUE + * {{NUMBEROFEDITS}} on en.wikipedia.org is now (January 2018) ~38% + * of {@code Integer.MAX_VALUE} * @param timestamp when this revision was made + * @param user the user making this revision (may be anonymous) + * @param comment the edit summary + * @param parsedcomment the edit summary, parsed into HTML * @param title the concerned article - * @param summary the edit summary - * @param user the user making this revision (may be anonymous, if not - * use User.getUsername()) + * @param sha1 the SHA-1 hash of the revision * @param minor whether this was a minor edit * @param bot whether this was a bot edit * @param rvnew whether this revision created a new page * @param size the size of the revision * @since 0.17 */ - public Revision(long revid, OffsetDateTime timestamp, String title, String summary, String user, - boolean minor, boolean bot, boolean rvnew, int size) + public Revision(long revid, OffsetDateTime timestamp, String user, String comment, + String parsedcomment, String title, String sha1, boolean minor, boolean bot, + boolean rvnew, int size) { - this.revid = revid; - this.timestamp = timestamp; - this.summary = summary; + super(revid, timestamp, user, Objects.requireNonNull(title), comment, parsedcomment); + this.sha1 = sha1; this.minor = minor; - this.user = user; - this.title = title; this.bot = bot; this.rvnew = rvnew; this.size = size; @@ -6447,7 +7249,7 @@ public Revision(long revid, OffsetDateTime timestamp, String title, String summa /** * Fetches the contents of this revision. - * @return the contents of the appropriate article at timestamp + * @return the contents of the appropriate article at timestamp * @throws IOException if a network error occurs * @throws IllegalArgumentException if page == Special:Log/xxx. * @since 0.17 @@ -6455,176 +7257,125 @@ public Revision(long revid, OffsetDateTime timestamp, String title, String summa public String getText() throws IOException { // logs have no content - if (revid < 1L) + if (getID() < 1L) throw new IllegalArgumentException("Log entries have no valid content!"); // TODO: returning a 404 here when revision content has been deleted // is not a good idea. if (pageDeleted) // FIXME: broken if a page is live, but has deleted revisions { - String url = query + "prop=deletedrevisions&drvprop=content&revids=" + revid; - String temp = fetch(url, "Revision.getText"); + Map getparams = new HashMap<>(); + getparams.put("action", "query"); + getparams.put("prop", "deletedrevisions"); + getparams.put("drvprop", "content"); + getparams.put("revids", String.valueOf(getID())); + String temp = makeApiCall(getparams, null, "Revision.getText"); + detectUncheckedErrors(temp, null, null); int a = temp.indexOf("", a) + 1; + a = temp.indexOf('>', a) + 1; int b = temp.indexOf("", a); // tag not present if revision has no content - log(Level.INFO, "Revision.getText", "Successfully retrieved text of revision " + revid); + log(Level.INFO, "Revision.getText", "Successfully retrieved text of revision " + getID()); return (b < 0) ? "" : temp.substring(a, b); } else - { - String url = base + encode(title, true) + "&oldid=" + revid + "&action=raw"; - String temp = fetch(url, "Revision.getText"); - log(Level.INFO, "Revision.getText", "Successfully retrieved text of revision " + revid); - return temp; - } + return Wiki.this.getText(null, new long[] { getID() }, -1).get(0); } /** * Gets the rendered text of this revision. * @return the rendered contents of the appropriate article at - * timestamp + * timestamp * @throws IOException if a network error occurs * @throws IllegalArgumentException if page == Special:Log/xxx. * @since 0.17 */ public String getRenderedText() throws IOException { - // logs have no content - if (revid < 1L) + if (getID() < 1L) throw new IllegalArgumentException("Log entries have no valid content!"); - - String temp; - if (pageDeleted) - { - String url = query + "prop=deletedrevisions&drvprop=content&drvparse=1&revids=" + revid; - temp = fetch(url, "Revision.getRenderedText"); - // TODO - } - else - { - String url = base + "&action=render&oldid=" + revid; - temp = fetch(url, "Revision.getRenderedText"); - } - log(Level.INFO, "Revision.getRenderedText", "Successfully retrieved rendered text of revision " + revid); - return decode(temp); + Map content = new HashMap<>(); + content.put("revision", this); + return Wiki.this.parse(content, -1, false); } /** - * Returns true if the revision content is RevisionDeleted. + * Returns the SHA-1 hash (base 16, lower case) of the content of this + * revision, or {@code null} if the revision content is RevisionDeleted + * and we cannot access it. + * + *

Warnings: + *

    + *
  • Not available through {@link #watchlist(RequestHelper)} or {@link + * #contribs(List, String, RequestHelper)}. + *
+ * * @return (see above) - * @since 0.31 + * @since 0.35 */ - public boolean isContentDeleted() + public String getSha1() { - return contentDeleted; + return sha1; } /** - * Returns a HTML rendered diff table; see the table at the example. - * Returns null for page creations, moves, protections and similar - * dummy edits ( - * example). - * + * Returns a HTML rendered diff table of this revision to other. + * See {@link #diff(Map, Map)} for full documentation. + * * @param other another revision on the same page. * @return the difference between this and the other revision * @throws IOException if a network error occurs + * @throws SecurityException if this or the other revision is + * RevisionDeleted and the user lacks the necessary privileges * @since 0.21 */ public String diff(Revision other) throws IOException { - return diff(other.revid, ""); + return Wiki.this.diff(Map.of("revision", this), Map.of("revision", other)); } /** - * Returns a HTML rendered diff table between this revision and the - * given text. Useful for emulating the "show changes" functionality. - * See the table at the text. Useful for emulating the "show changes" + * functionality. See the table at the example. + * * @param text some wikitext * @return the difference between this and the the text provided * @throws IOException if a network error occurs + * @throws SecurityException if this or the other revision is + * RevisionDeleted and the user lacks the necessary privileges * @since 0.21 */ public String diff(String text) throws IOException { - return diff(0L, text); + return Wiki.this.diff(Map.of("revision", this), Map.of("text", text)); } /** - * Returns a HTML rendered diff table; see the table at the example. - * Returns null for page creations, moves, protections and - * similar dummy edits (example) - * and pairs of revisions where there is no difference. - * - * @param oldid the oldid of a revision on the same page. {@link Wiki#NEXT_REVISION}, - * {@link Wiki#PREVIOUS_REVISION} and {@link Wiki#CURRENT_REVISION} can - * be used here for obvious effect. + * Returns a HTML rendered diff table from this revision to the given + * oldid. See {@link #diff(Map, Map)} for full documentation. + * + * @param oldid the oldid of a revision on the same page. {@link + * Wiki#NEXT_REVISION}, {@link Wiki#PREVIOUS_REVISION} and {@link + * Wiki#CURRENT_REVISION} can be used here for obvious effect. * @return the difference between this and the other revision * @throws IOException if a network error occurs + * @throws SecurityException if this or the other revision is + * RevisionDeleted and the user lacks the necessary privileges * @since 0.26 */ public String diff(long oldid) throws IOException { - return diff(oldid, ""); - } - - /** - * Fetches a HTML rendered diff table; see the table at the example. - * Returns null for page creations, moves, protections and similar - * dummy edits ( - * example) and pairs of revisions where there is no difference. - * - * @param oldid the id of another revision; (exclusive) or - * @param text some wikitext to compare against - * @return a difference between oldid or text or null if there is no - * diff. - * @throws IOException if a network error occurs - * @since 0.21 - */ - protected String diff(long oldid, String text) throws IOException - { - StringBuilder temp = new StringBuilder(); - if (oldid == NEXT_REVISION) - temp.append("&torelative=next"); - else if (oldid == CURRENT_REVISION) - temp.append("&torelative=cur"); - else if (oldid == PREVIOUS_REVISION) - temp.append("&torelative=prev"); - else if (oldid == 0L) - { - temp.append("&totext="); - temp.append(text); - } - else - { - temp.append("&torev="); - temp.append(oldid); - } - - String line = post(apiUrl + "action=compare&fromrev=" + getRevid(), temp.toString(), "Revision.diff"); - - // strip extraneous information - if (line.contains("")) - { - int a = line.indexOf("", a) + 1; - int b = line.indexOf("", a); - return decode(line.substring(a, b)); - } - else - // tag has no content if there is no diff or the two - // revisions are identical. In particular, the API does not - // distinguish between: - // https://en.wikipedia.org/w/index.php?title=Source_Filmmaker&diff=804972897&oldid=803731343 (no difference) - // https://en.wikipedia.org/w/index.php?title=Dayo_Israel&oldid=738178354&diff=prev (dummy edit) - return null; + Map from = new HashMap<>(); + from.put("revision", this); + Map to = new HashMap<>(); + to.put("revid", oldid); + return Wiki.this.diff(from, to); } /** - * Determines whether this Revision is equal to another object. + * Determines whether this Revision is equal to another based on the + * underlying {@linkplain Event#equals(Object) Event}. * @param o an object * @return whether o is equal to this object * @since 0.17 @@ -6632,20 +7383,29 @@ else if (oldid == 0L) @Override public boolean equals(Object o) { + // Note to self: don't use SHA-1 until all API calls provide it. + if (!super.equals(o)) + return false; if (!(o instanceof Revision)) return false; - return revid == ((Revision)o).revid; + // Revision rev = (Revision)o; + // return Objects.equals(sha1, rev.sha1); + return true; } /** - * Returns a hash code of this revision. + * Returns a hash code of this revision based on the underlying + * {@linkplain Event#hashCode() Event}. * @return a hash code * @since 0.17 */ @Override public int hashCode() { - return (int)revid * 2 - Wiki.this.hashCode(); + // Note to self: don't use SHA-1 until all API calls provide it. + int hc = super.hashCode(); + hc = 127 * hc; + return hc; } /** @@ -6671,12 +7431,15 @@ public boolean isBot() } /** - * Determines whether this revision created a new page.
- * WARNING: Will return false for all revisions prior to 2007 - * (I think?) -- this is a MediaWiki problem.
- * WARNING: Returning true does not imply this is the bottommost - * revision on the page due to histmerges.
- * WARNING: Not accessible through getPageHistory() -- a MW problem. + * Determines whether this revision created a new page. + * + *

Warnings: + *

    + *
  • Returning {@code true} does not imply this is the bottommost + * revision on the page due to histmerges. + *
  • Not available through {@link #getPageHistory(String, Wiki.RequestHelper)} + *
+ * * @return (see above) * @since 0.27 */ @@ -6686,51 +7449,8 @@ public boolean isNew() } /** - * Returns the edit summary for this revision, or null if the summary - * was RevisionDeleted and you lack the necessary privileges. - * @return the edit summary - * @since 0.17 - */ - public String getSummary() - { - return summary; - } - - /** - * Returns true if the edit summary is RevisionDeleted. - * @return (see above) - * @since 0.30 - */ - public boolean isSummaryDeleted() - { - return summaryDeleted; - } - - /** - * Returns the user or anon who created this revision. You should - * pass this (if not an IP) to getUser(String) to obtain a - * User object. Returns null if the user was RevisionDeleted and you - * lack the necessary privileges. - * @return the user or anon - * @since 0.17 - */ - public String getUser() - { - return user; - } - - /** - * Returns true if the user is RevisionDeleted. - * @return (see above) - * @since 0.30 - */ - public boolean isUserDeleted() - { - return userDeleted; - } - - /** - * Returns true if this revision is deleted (different from revdeleted). + * Returns {@code true} if this revision is deleted (not the same as + * RevisionDeleted). * @return (see above) * @since 0.31 */ @@ -6739,37 +7459,6 @@ public boolean isPageDeleted() return pageDeleted; } - /** - * Returns the page to which this revision was made. - * @return the page - * @since 0.17 - */ - public String getPage() - { - return title; - } - - /** - * Returns the oldid of this revision. Don't confuse this with - * rcid - * @return the oldid (long) - * @since 0.17 - */ - public long getRevid() - { - return revid; - } - - /** - * Gets the time that this revision was made. - * @return the timestamp - * @since 0.17 - */ - public OffsetDateTime getTimestamp() - { - return timestamp; - } - /** * Gets the size of this revision in bytes. * @return see above @@ -6781,7 +7470,8 @@ public int getSize() } /** - * Returns the change in page size caused by this revision. + * Returns the change in page size caused by this revision. Not available + * through getPageHistory or getDeletedHistory. * @return see above * @since 0.28 */ @@ -6798,22 +7488,8 @@ public int getSizeDiff() @Override public String toString() { - StringBuilder sb = new StringBuilder("Revision[oldid="); - sb.append(revid); - sb.append(",page=\""); - sb.append(title); - sb.append("\",user="); - sb.append(user == null ? "[hidden]" : user); - sb.append(",userdeleted="); - sb.append(userDeleted); - sb.append(",timestamp="); - sb.append(timestamp.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - sb.append(",summary=\""); - sb.append(summary == null ? "[hidden]" : summary); - sb.append("\",summarydeleted="); - sb.append(summaryDeleted); - sb.append(",contentDeleted="); - sb.append(contentDeleted); + StringBuilder sb = new StringBuilder(super.toString()); + sb.deleteCharAt(sb.length() - 1); sb.append(",minor="); sb.append(minor); sb.append(",bot="); @@ -6826,25 +7502,10 @@ public String toString() sb.append(previous); sb.append(",next="); sb.append(next); - sb.append(",rollbacktoken="); - sb.append(rollbacktoken == null ? "null" : rollbacktoken); sb.append("]"); return sb.toString(); } - /** - * Compares this revision to another revision based on the recentness - * of their timestamps. - * @param other the revision to compare - * @return whether this object is equal to - * @since 0.18 - */ - @Override - public int compareTo(Wiki.Revision other) - { - return timestamp.compareTo(timestamp); - } - /** * Gets the previous revision. * @return the previous revision, or null if this is the first revision @@ -6892,35 +7553,25 @@ public long getRcid() } /** - * Sets a rollback token for this revision. - * @param token a rollback token - * @since 0.24 - */ - public void setRollbackToken(String token) - { - rollbacktoken = token; - } - - /** - * Gets the rollback token for this revision. Can be null, and often - * for good reasons: cannot rollback or not top revision. - * @return the rollback token - * @since 0.24 + * Gets a permanent URL to the human-readable version of this Revision. + * This URL uses index.php, not Special:Permanentlink for ease of adding + * other parameters. + * @return (see above) + * @since 0.35 */ - public String getRollbackToken() + public String permanentUrl() { - return rollbacktoken; + return getIndexPhpUrl() + "?oldid=" + getID(); } /** - * Reverts this revision using the rollback method. See - * {@link Wiki#rollback(org.wikipedia.Wiki.Revision)}. - * + * Reverts this revision using the rollback method. + * * @throws IOException if a network error occurs - * @throws CredentialNotFoundException if not logged in or user is not - * an admin + * @throws SecurityException if the user lacks the privileges to rollback * @throws CredentialExpiredException if cookies have expired * @throws AccountLockedException if the user is blocked + * @see Wiki#rollback(org.wikipedia.Wiki.Revision) * @since 0.19 */ public void rollback() throws IOException, LoginException @@ -6929,174 +7580,588 @@ public void rollback() throws IOException, LoginException } /** - * Reverts this revision using the rollback method. See - * {@link Wiki#rollback(org.wikipedia.Wiki.Revision)}. - * + * Reverts this revision using the rollback method. + * * @param bot mark this and the reverted revision(s) as bot edits * @param reason (optional) a custom reason * @throws IOException if a network error occurs - * @throws CredentialNotFoundException if not logged in or user is not - * an admin + * @throws SecurityException if the user lacks the privileges to rollback * @throws CredentialExpiredException if cookies have expired * @throws AccountLockedException if the user is blocked + * @see Wiki#rollback(org.wikipedia.Wiki.Revision) * @since 0.19 */ - public void rollback(boolean bot, String reason) throws IOException, LoginException + public void rollback(boolean bot, String reason) throws IOException, LoginException + { + Wiki.this.rollback(this, bot, reason); + } + } + + /** + * Vehicle for stuffing standard optional parameters into Wiki queries. + * {@code RequestHelper} objects are reusable. The following example + * fetches articles from the back of the new pages queue on the + * English Wikipedia. + * + * {@code 
+     *  Wiki.RequestHelper rh = enWiki.new RequestHelper()
+     *      .inNamespaces(Wiki.MAIN_NAMESPACE)
+     *      .reverse();
+     *  List newpages = enWiki.newPages(rh);
+     *
} + * + * @since 0.36 + */ + public class RequestHelper + { + private String title; + private String byuser; + private OffsetDateTime earliest, latest; + private int[] localns = new int[0]; + private boolean reverse = false; + private String notbyuser; + private String tag; + private Map options; + private String requestType; + private int limit = -1; + + /** + * Creates a new RequestHelper. + */ + public RequestHelper() + { + } + + /** + * Limits query results to Events occuring on the given title. If a + * query mandates a title parameter (e.g. {@link #getPageHistory(String, + * RequestHelper)}, don't use this. Use the parameter in the query + * method instead. + * @param title a page title + * @return this RequestHelper + */ + public RequestHelper byTitle(String title) + { + this.title = (title == null) ? null : normalize(title); + return this; + } + + /** + * Limits query results to Events triggered by the given user. If a query + * mandates a user parameter (e.g. {@link #contribs(List, String, RequestHelper)}, + * don't use this. Use the parameter in the query method instead. + * @param byuser some username or IP address + * @return this RequestHelper + */ + public RequestHelper byUser(String byuser) + { + this.byuser = (byuser == null) ? null : normalize(byuser); + return this; + } + + /** + * Limit results to be within this date range. + * @param earliest the lower (earliest) date bound, use {@code null} to + * not set one + * @param latest the higher (latest) date bound, use {@code null} to + * not set one + * @throws IllegalArgumentException if {@code earliest.isAfter(latest)} + * @return this RequestHelper + */ + public RequestHelper withinDateRange(OffsetDateTime earliest, OffsetDateTime latest) + { + if (earliest != null && latest != null && earliest.isAfter(latest)) + throw new IllegalArgumentException("Earliest date must be before latest date!"); + this.earliest = earliest; + this.latest = latest; + return this; + } + + /** + * Limits query results to the given namespaces. + * @param ns a list of namespaces + * @return this RequestHelper + */ + public RequestHelper inNamespaces(int... ns) + { + localns = ns; + return this; + } + + /** + * Should we perform this query in reverse order (earliest first). + * @param reverse whether to reverse this query + * @return this RequestHelper + */ + public RequestHelper reverse(boolean reverse) + { + this.reverse = reverse; + return this; + } + + /** + * Limits query results to {@link Event Events} that have been tagged + * with the given tag. + * @param tag a change tag + * @return this RequestHelper + */ + public RequestHelper taggedWith(String tag) + { + this.tag = tag; + return this; + } + + /** + * Limits query results to Events NOT triggered by the given user. + * @param notbyuser some username or IP address to exclude + * @return this RequestHelper + */ + public RequestHelper notByUser(String notbyuser) + { + this.notbyuser = (notbyuser == null) ? null : normalize(notbyuser); + return this; + } + + /** + * Return no more than the given number of results. Overrides {@linkplain + * #setQueryLimit(int) global limits}. Supply a negative integer to use + * global limits. + * @param limit a positive integer + * @return this RequestHelper + */ + public RequestHelper limitedTo(int limit) + { + this.limit = limit; + return this; + } + + /** + * Filters a set of returned results using the given options. Please + * check calling method documentation for supported options. + * + *

+ * When filtering revisions, available keys may include "minor", "top", + * "new", "bot", "anon", "redirect", "patrolled" and "unread" for + * vanilla MediaWiki. Extensions may define their own. For instance, + * {@code rcoptions = { minor = true, anon = false, patrolled = false}} + * returns all minor edits from logged in users that aren't patrolled. + * Setting "patrolled" limits results to no older than retention in + * the recentchanges + * table. + * + * @param options the options to filter by + * @return this RequestHelper + */ + public RequestHelper filterBy(Map options) + { + this.options = options; + return this; + } + + /** + * Sets the prefix of API request parameters (the XX in XXlimit, XXdir, + * XXnamespace and so forth). Internal use only. + * @param prefix the prefix to use (must not be null) + */ + protected void setRequestType(String prefix) + { + requestType = Objects.requireNonNull(prefix); + } + + /** + * Returns a HTTP request parameter containing the title to get + * events for, or an empty map if not wanted. + * @return (see above) + */ + protected Map addTitleParameter() + { + if (title != null) + return Map.of(requestType + "title", title); + return Collections.emptyMap(); + } + + /** + * Returns a HTTP request parameter containing the user to filter + * returned events by, or an empty map if not wanted. + * @return (see above) + */ + protected Map addUserParameter() + { + if (byuser != null) + return Map.of(requestType + "user", byuser); + return Collections.emptyMap(); + } + + /** + * Returns a HTTP request parameter containing the dates to start + * and end enumeration, or an empty map if not wanted. + * @return (see above) + */ + protected Map addDateRangeParameters() + { + // https://phabricator.wikimedia.org/T16449 + Map temp = new HashMap<>(); + OffsetDateTime odt = reverse ? earliest : latest; + if (odt != null) + temp.put(requestType + "start", + // https://www.mediawiki.org/wiki/Timestamp + // https://github.com/MER-C/wiki-java/issues/170 + odt.withOffsetSameInstant(ZoneOffset.UTC) + .truncatedTo(ChronoUnit.MICROS) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + odt = reverse ? latest : earliest; + if (odt != null) + temp.put(requestType + "end", + // https://www.mediawiki.org/wiki/Timestamp + // https://github.com/MER-C/wiki-java/issues/170 + odt.withOffsetSameInstant(ZoneOffset.UTC) + .truncatedTo(ChronoUnit.MICROS) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + return temp; + } + + /** + * Returns a HTTP request parameter containing the namespaces to limit + * this query to, or an empty map if not wanted. + * @return (see above) + */ + protected Map addNamespaceParameter() + { + if (localns.length != 0) + return Map.of(requestType + "namespace", constructNamespaceString(localns)); + return Collections.emptyMap(); + } + + /** + * Returns a HTTP request parameter instructing the API to reverse the + * query, or an empty map if not wanted. + * @return (see above) + */ + protected Map addReverseParameter() + { + return Map.of(requestType + "dir", reverse ? "newer" : "older"); + } + + /** + * Returns a HTTP request parameter containing the tag to limit + * returned events to, or an empty map if not wanted. + * @return (see above) + */ + protected Map addTagParameter() + { + if (tag != null) + return Map.of(requestType + "tag", tag); + return Collections.emptyMap(); + } + + /** + * Returns a HTTP request parameter containing the user to exclude + * when returning events, or an empty map if not wanted. + * @return (see above) + */ + protected Map addExcludeUserParameter() + { + if (notbyuser != null) + return Map.of(requestType + "excludeuser", notbyuser); + return Collections.emptyMap(); + } + + /** + * Returns HTTP request parameter(s) containing flags to filter returned + * revisions by, or an empty map if not wanted. + * @return (see above) + */ + protected Map addShowParameter() + { + Map temp = new HashMap<>(); + if (options != null && !options.isEmpty()) + { + // deal with MW API annoyance for action=query&list=watchlist - see watchlist(rh) + Boolean top = null; + if (requestType.equals("wl")) + { + top = options.remove("top"); + if (Boolean.TRUE.equals(top)) + temp.put("wlallrev", "1"); + } + + StringBuilder sb = new StringBuilder(); + options.forEach((key, value) -> + { + if (Boolean.FALSE.equals(value)) + sb.append('!'); + sb.append(key); + sb.append("|"); + }); + temp.put(requestType + "show", sb.substring(0, sb.length() - 1)); + + if (top != null) // put it back + options.put("top", top); + } + return temp; + } + + /** + * Returns the number of results the query should be limited to. If not + * present, use {@linkplain #setQueryLimit(int) global limits}. + * @return (see above) + */ + public int limit() { - Wiki.this.rollback(this, bot, reason); + return limit; } } // INTERNALS + /** + * Performs a vectorized action=query&prop=X type API query + * over titles. + * @param queryPrefix the request type prefix (e.g. "pl" for prop=links) + * @param getparams a bunch of parameters to send via HTTP GET + * @param titles a list of titles + * @param caller the name of the calling method + * @param limit fetch no more than this many results + * @param parser a BiConsumer that parses the XML returned by the MediaWiki + * API into things we want, dumping them into the given List + * @return a list of results, where each element corresponds to the element + * at the same index in the input title list + * @since 0.36 + * @throws IOException if a network error occurs + */ + protected List> makeVectorizedQuery(String queryPrefix, Map getparams, + List titles, String caller, int limit, BiConsumer> parser) throws IOException + { + // copy because normalization and redirect resolvers overwrite + List titles2 = new ArrayList<>(titles); + List>> stuff = new ArrayList<>(); + Map postparams = new HashMap<>(); + for (String temp : constructTitleString(titles2)) + { + postparams.put("titles", temp); + stuff.addAll(makeListQuery(queryPrefix, getparams, postparams, caller, -1, (line, results) -> + { + // Split the result into individual listings for each article. + String[] x = line.split(" list = new ArrayList<>(); + parser.accept(x[i], list); + + Map> intermediate = new HashMap<>(); + intermediate.put(parsedtitle, list); + results.add(intermediate); + } + })); + } + + // prepare the return list + List> ret = Stream.generate(() -> new ArrayList()) + .limit(titles2.size()) + .collect(Collectors.toCollection(ArrayList::new)); + // then retrieve the results from the intermediate list of maps, + // ensuring results correspond to inputs + stuff.forEach(map -> + { + String parsedtitle = map.keySet().iterator().next(); + List templates = map.get(parsedtitle); + for (int i = 0; i < titles2.size(); i++) + if (titles2.get(i).equals(parsedtitle)) + ret.get(i).addAll(templates); + }); + return ret; + } + /** * Fetches list-type results from the MediaWiki API. - * - * @param a class describing the parsed API results (e.g. String, + * + * @param a class describing the parsed API results (e.g. String, * LogEntry, Revision) * @param queryPrefix the request type prefix (e.g. "pl" for prop=links) - * @param url the query URL, without the limit and XXcontinue parameters + * @param getparams a bunch of parameters to send via HTTP GET + * @param postparams if not null, send these parameters via POST (see + * {@link #makeApiCall(Map, Map, String) }). * @param caller the name of the calling method + * @param limit fetch no more than this many results * @param parser a BiConsumer that parses the XML returned by the MediaWiki * API into things we want, dumping them into the given List * @return the query results * @throws IOException if a network error occurs + * @throws SecurityException if we don't have the credentials to perform a + * privileged action (mostly avoidable) * @since 0.34 */ - protected List queryAPIResult(String queryPrefix, StringBuilder url, String caller, - BiConsumer> parser) throws IOException + protected List makeListQuery(String queryPrefix, Map getparams, + Map postparams, String caller, int limit, BiConsumer> parser) throws IOException { + if (limit < 0) + limit = querylimit; + getparams.put("action", "query"); List results = new ArrayList<>(1333); - StringBuilder xxcontinue = new StringBuilder(); - url.append("&"); - url.append(queryPrefix); - url.append("limit="); + String limitstring = queryPrefix + "limit"; do { - int limit = Math.min(querylimit - results.size(), max); - String tempurl = url.toString() + limit; - String line = fetch(tempurl + xxcontinue.toString(), caller); - xxcontinue.setLength(0); - + getparams.put(limitstring, String.valueOf(Math.min(limit - results.size(), max))); + String line = makeApiCall(getparams, postparams, caller); + detectUncheckedErrors(line, null, null); + getparams.keySet().removeIf(param -> param.endsWith("continue")); + // Continuation parameter has form: // if (line.contains("", a); - String[] temp = line.substring(a, b).split("\""); - xxcontinue.append("&"); - xxcontinue.append(temp[0]); - xxcontinue.append(encode(temp[1], false)); - xxcontinue.append(temp[2].replace(" ", "&")); - xxcontinue.append(encode(temp[3], false)); + int a = line.indexOf("", a); + String cont = line.substring(a, b); + for (String contpair : cont.split("\" ")) + { + contpair = " " + contpair.trim(); + String contattr = contpair.substring(0, contpair.indexOf("=\"")); + getparams.put(contattr.trim(), parseAttribute(cont, contattr, 0)); + } } - + parser.accept(line, results); } - while (xxcontinue.length() != 0 && results.size() < querylimit); + while (getparams.containsKey("continue") && results.size() < limit); return results; } - - // miscellany + // miscellany + + /** + * Convenience method for checking user permissions. + * @param right a user rights + * @param morerights additional user rights + * @throws SecurityException if the permission check fails + * @since 0.37 + */ + protected void checkPermissions(String action, String right, String... morerights) + { + if (user == null) + throw new SecurityException("Cannot " + action + ": not logged in."); + if (!user.isAllowedTo(right, morerights)) + throw new SecurityException("Cannot " + action + ": permission denied."); + } + /** - * A generic URL content fetcher. This is only useful for GET requests, - * which is almost everything that doesn't modify the wiki. Might be - * useful for subclasses. + * Constructs, sends and handles calls to the MediaWiki API. This is a + * low-level method for making your own, custom API calls. + * + *

+ * If postparams is not {@code null} or empty, the request is + * sent using HTTP GET, otherwise it is sent using HTTP POST. A + * {@code byte[]} value in postparams causes the request to be + * sent as a multipart POST. Anything else is converted to String via the + * following means: + * + *

    + *
  • String[] -- {@code String.join("|", arr)} + *
  • StringBuilder -- {@code sb.toString()} + *
  • Number -- {@code num.toString()} + *
  • OffsetDateTime -- {@code date.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)} + *
  • {@code Collection} -- {@code coll.stream() + * .map(item -> convertToString(item)) // using the above rules + * .collect(Collectors.joining("|"))} + *
+ * + *

+ * All supplied Strings and objects converted to String are automatically + * URLEncoded in UTF-8 if this is a normal POST request. * + *

* Here we also check the database lag and wait if it exceeds * maxlag, see * here for how this works. * - * @param url the url to fetch + * @param getparams append these parameters to the urlbase + * @param postparams if null, send the request using POST otherwise use GET * @param caller the caller of this method - * @return the content of the fetched URL + * @return the server response * @throws IOException if a network error occurs + * @throws SecurityException if we don't have the credentials to perform a + * privileged action (mostly avoidable) * @throws AssertionError if assert=user|bot fails + * @see Multipart/form-data * @since 0.18 */ - protected String fetch(String url, String caller) throws IOException + public String makeApiCall(Map getparams, Map postparams, String caller) throws IOException { - String temp = null; - int tries = maxtries; - do + // build the URL + StringBuilder urlbuilder = new StringBuilder(apiUrl + "?"); + getparams.putAll(defaultApiParams); + for (Map.Entry entry : getparams.entrySet()) { - logurl(url, caller); - tries--; - try + urlbuilder.append('&'); + urlbuilder.append(entry.getKey()); + urlbuilder.append('='); + urlbuilder.append(URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8)); + } + String url = urlbuilder.toString(); + + // POST stuff + boolean isPOST = (postparams != null && !postparams.isEmpty()); + StringBuilder stringPostBody = new StringBuilder(); + boolean multipart = false; + ArrayList multipartPostBody = new ArrayList<>(); + String boundary = "----------NEXT PART----------"; + if (isPOST) + { + // determine whether this is a multipart post and convert any values + // to String if necessary + for (Map.Entry entry : postparams.entrySet()) { - // connect - URLConnection connection = makeConnection(url); - connection.setConnectTimeout(CONNECTION_CONNECT_TIMEOUT_MSEC); - connection.setReadTimeout(CONNECTION_READ_TIMEOUT_MSEC); - setCookies(connection); - connection.connect(); - grabCookies(connection); - - // check lag and retry - if (checkLag(connection)) - return fetch(url, caller); - - // get the text - String line; - StringBuilder text = new StringBuilder(100000); - try (BufferedReader in = new BufferedReader(new InputStreamReader( - zipped ? new GZIPInputStream(connection.getInputStream()) : connection.getInputStream(), "UTF-8"))) + Object value = entry.getValue(); + if (value instanceof byte[]) + multipart = true; + else + entry.setValue(convertToString(value)); + } + + // now we know how we're sending it, construct the post body + if (multipart) + { + byte[] nextpart = ("--" + boundary + "\r\n\"Content-Disposition: form-data; name=\\\"\"") + .getBytes(StandardCharsets.UTF_8); + for (Map.Entry entry : postparams.entrySet()) { - while ((line = in.readLine()) != null) + multipartPostBody.add(nextpart); + Object value = entry.getValue(); + multipartPostBody.add((entry.getKey() + "\"\r\n").getBytes(StandardCharsets.UTF_8)); + if (value instanceof String) + multipartPostBody.add(("Content-Type: text/plain; charset=UTF-8\r\n\r\n" + (String)value + "\r\n") + .getBytes(StandardCharsets.UTF_8)); + else if (value instanceof byte[]) { - text.append(line); - text.append("\n"); + multipartPostBody.add("Content-Type: application/octet-stream\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + multipartPostBody.add((byte[])value); + multipartPostBody.add("\r\n".getBytes(StandardCharsets.UTF_8)); } } - temp = text.toString(); + multipartPostBody.add((boundary + "--\r\n").getBytes(StandardCharsets.UTF_8)); } - catch (IOException ex) + else { - if (tries == 0) - throw ex; - try - { - Thread.sleep(10000); - } - catch (InterruptedException ignored) + // automatically encode Strings sent via normal POST + for (Map.Entry entry : postparams.entrySet()) { + stringPostBody.append('&'); + stringPostBody.append(entry.getKey()); + stringPostBody.append('='); + stringPostBody.append(URLEncoder.encode(entry.getValue().toString(), StandardCharsets.UTF_8)); } } } - while (temp == null); - if (temp.contains(" hr = client.send(connection.build(), HttpResponse.BodyHandlers.ofInputStream()); + boolean zipped_ = hr.headers().firstValue("Content-Encoding").orElse("").equals("gzip"); + if (checkLag(hr)) { - out.write(text); + tries++; + throw new HttpRetryException("Database lagged.", 503); } - // check lag and retry - if (checkLag(connection)) - return post(url, text, caller); - grabCookies(connection); - StringBuilder buffer = new StringBuilder(100000); - String line; try (BufferedReader in = new BufferedReader(new InputStreamReader( - zipped ? new GZIPInputStream(connection.getInputStream()) : connection.getInputStream(), "UTF-8"))) + zipped_ ? new GZIPInputStream(hr.body()) : hr.body(), "UTF-8"))) { - while ((line = in.readLine()) != null) - { - buffer.append(line); - buffer.append("\n"); - } + response = in.lines().collect(Collectors.joining("\n")); } - temp = buffer.toString(); - - // check for recoverable errors - - // rate limit (though might be a long one e.g. email) - if (temp.contains("error code=\"ratelimited\"")) + + // Check for rate limit (though might be a long one e.g. email) + if (response.contains("error code=\"ratelimited\"")) { + // the Retry-After header field is useless here + // see https://phabricator.wikimedia.org/T172293 log(Level.WARNING, caller, "Server-side throttle hit."); + Thread.sleep(10000); throw new HttpRetryException("Action throttled.", 503); } - // database lock - if (temp.contains("error code=\"readonly\"")) + // Check for database lock + if (response.contains("error code=\"readonly\"")) { log(Level.WARNING, caller, "Database locked!"); + Thread.sleep(10000); throw new HttpRetryException("Database locked!", 503); } - return temp; + + // No need to retry anymore, success or unrecoverable failure. + tries = 0; } catch (IOException ex) { + // Exception deliberately ignored until retries are depleted. if (tries == 0) throw ex; - try - { - Thread.sleep(10000); - } - catch (InterruptedException ignored) - { - } + } + catch (InterruptedException ignored) + { } } - while (temp.isEmpty()); - throw new AssertionError("Unreachable."); + while (tries != 0); + + // empty response from server + if (response.isEmpty()) + throw new UnknownError("Received empty response from server!"); + return response; } /** - * Performs a multi-part HTTP POST. - * @param url the url to post to - * @param params the POST parameters. Supported types: UTF-8 text, byte[]. - * Text and parameter names must NOT be URL encoded. - * @param caller the caller of this method - * @return the server response - * @throws IOException if a network error occurs - * @see #post(java.lang.String, java.lang.String, java.lang.String) - * @see Multipart/form-data - * @since 0.27 + * Converts HTTP POST parameters to Strings. See {@link #makeApiCall(Map, + * Map, String)} for the description. + * @param param the parameter to convert + * @return that parameter, as a String + * @throws UnsupportedOperationException if param is not a supported data type + * @since 0.35 */ - protected String multipartPost(String url, Map params, String caller) throws IOException + private String convertToString(Object param) { - String temp = ""; - int tries = maxtries; - do + // TODO: Replace with type switch in JDK 11/12 + if (param instanceof String) + return (String)param; + else if (param instanceof StringBuilder || param instanceof Number) + return param.toString(); + else if (param instanceof String[]) + return String.join("|", (String[])param); + else if (param instanceof OffsetDateTime) { - logurl(url, caller); - tries--; - try - { - URLConnection connection = makeConnection(url); - String boundary = "----------NEXT PART----------"; - connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary); - setCookies(connection); - connection.setDoOutput(true); - connection.setConnectTimeout(CONNECTION_CONNECT_TIMEOUT_MSEC); - connection.setReadTimeout(CONNECTION_READ_TIMEOUT_MSEC); - connection.connect(); - - // write stuff to a local buffer - boundary = "--" + boundary + "\r\n"; - ByteArrayOutputStream bout = new ByteArrayOutputStream(); - try (DataOutputStream out = new DataOutputStream(bout)) - { - out.writeBytes(boundary); - - // write params - for (Map.Entry entry : params.entrySet()) - { - String name = entry.getKey(); - Object value = entry.getValue(); - out.writeBytes("Content-Disposition: form-data; name=\"" + name + "\"\r\n"); - if (value instanceof String) - { - out.writeBytes("Content-Type: text/plain; charset=UTF-8\r\n\r\n"); - out.write(((String)value).getBytes("UTF-8")); - } - else if (value instanceof byte[]) - { - out.writeBytes("Content-Type: application/octet-stream\r\n\r\n"); - out.write((byte[])value); - } - else - throw new UnsupportedOperationException("Unrecognized data type"); - out.writeBytes("\r\n"); - out.writeBytes(boundary); - } - out.writeBytes("--\r\n"); - } - try (OutputStream uout = connection.getOutputStream()) - { - // write the buffer to the URLConnection - uout.write(bout.toByteArray()); - } - - // check lag and retry - if (checkLag(connection)) - return multipartPost(url, params, caller); - - // done, read the response - grabCookies(connection); - String line; - StringBuilder buffer = new StringBuilder(100000); - try (BufferedReader in = new BufferedReader(new InputStreamReader( - zipped ? new GZIPInputStream(connection.getInputStream()) : connection.getInputStream(), "UTF-8"))) - { - while ((line = in.readLine()) != null) - { - buffer.append(line); - buffer.append("\n"); - } - } - temp = buffer.toString(); - - // check for recoverable errors - - // rate limit (though might be a long one e.g. email) - if (temp.contains("error code=\"ratelimited\"")) - { - log(Level.WARNING, caller, "Server-side throttle hit."); - throw new HttpRetryException("Action throttled.", 503); - } - // database lock - if (temp.contains("error code=\"readonly\"")) - { - log(Level.WARNING, caller, "Database locked!"); - throw new HttpRetryException("Database locked!", 503); - } - return temp; - } - catch (IOException ex) - { - if (tries == 0) - throw ex; - try - { - Thread.sleep(10000); - } - catch (InterruptedException ignored) - { - } - } + OffsetDateTime date = (OffsetDateTime)param; + // https://www.mediawiki.org/wiki/Timestamp + // https://github.com/MER-C/wiki-java/issues/170 + return date.atZoneSameInstant(ZoneOffset.UTC) + .truncatedTo(ChronoUnit.MICROS) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); } - while (temp.isEmpty()); - throw new AssertionError("Unreachable."); + else if (param instanceof Collection) + { + Collection coll = (Collection)param; + return coll.stream() + .map(item -> convertToString(item)) + .collect(Collectors.joining("|")); + } + else + throw new UnsupportedOperationException("Unrecognized data type"); } - + /** - * Checks for database lag and sleeps if lag > maxlag. - * @param connection the URL connection used in the request + * Checks for database lag and sleeps if {@code lag < getMaxLag()}. + * @param response the HTTP response received * @return true if there was sufficient database lag. + * @throws InterruptedException if any wait was interrupted + * @see #getMaxLag() + * @see + * MediaWiki documentation * @since 0.32 */ - protected synchronized boolean checkLag(URLConnection connection) + protected synchronized boolean checkLag(HttpResponse response) throws InterruptedException { - int lag = connection.getHeaderFieldInt("X-Database-Lag", -5); + HttpHeaders hdrs = response.headers(); + long lag = hdrs.firstValueAsLong("X-Database-Lag").orElse(-5); // X-Database-Lag is the current lag rounded down to the nearest integer. // Thus, we need to retry in case of equality. if (lag >= maxlag) { - try - { - int time = connection.getHeaderFieldInt("Retry-After", 10); - logger.log(Level.WARNING, "Current database lag {0} s exceeds maxlag of {1} s, waiting {2} s.", new Object[] { lag, maxlag, time }); - Thread.sleep(time * 1000); - } - catch (InterruptedException ignored) - { - } + long time = hdrs.firstValueAsLong("Retry-After").orElse(10); + logger.log(Level.WARNING, "Current database lag {0} s exceeds maxlag of {1} s, waiting {2} s.", new Object[] { lag, maxlag, time }); + Thread.sleep(time * 1000L); return true; } return false; } /** - * Creates a new URL connection. Override to change SSL handling, use a - * proxy, etc. + * Sets the HTTPClient used by this instance. Use this to set a proxy, + * SSL parameters and the connection timeout. + * @param builder a HTTP request builder + * @since 0.37 + */ + public void setHttpClient(HttpClient.Builder builder) + { + client = builder.cookieHandler(cookies).build(); + } + + /** + * Creates a new HTTP request. Override to change request properties. * @param url a URL string - * @return a connection to that URL + * @return a HTTP request builder for that URL * @throws IOException if a network error occurs * @since 0.31 */ - protected URLConnection makeConnection(String url) throws IOException + protected HttpRequest.Builder makeConnection(String url) throws IOException { - return new URL(url).openConnection(); + return HttpRequest.newBuilder(URI.create(url)) + .timeout(Duration.ofMillis(read_timeout_msec)) + .header("User-Agent", useragent) + .header("Accept-encoding", "gzip"); } + /** + * Checks for errors from standard read/write requests and throws the + * appropriate unchecked exception. + * + * @param response the response from the server to analyze + * @param errors additional errors to check for where throwing an unchecked + * exception is the desired behavior (function is of MediaWiki error + * message) + * @param warnings additional errors to check for where throwing an exception + * is not required (function is of MediaWiki error message) + * @throws RuntimeException or subclasses depending on the type of errors + * checked for + * @return whether the action was successful + * @since 0.37 + */ + protected boolean detectUncheckedErrors(String response, Map> errors, + Map> warnings) + { + if (response.contains("> uncheckederrors, + Map> info) throws IOException, LoginException { // perform various status checks every 100 or so edits if (statuscounter > statusinterval) { // purge user rights in case of desysop or loss of other priviliges - user.getUserInfo(); + user = getUsers(List.of(user.getUsername())).get(0); if ((assertion & ASSERT_SYSOP) == ASSERT_SYSOP && !user.isA("sysop")) // assert user.isA("sysop") : "Sysop privileges missing or revoked, or session expired"; throw new AssertionError("Sysop privileges missing or revoked, or session expired"); @@ -7358,35 +8408,36 @@ protected void checkErrorsAndUpdateStatus(String line, String caller) throws IOE else statuscounter++; - // successful - if (line.contains("result=\"Success\"")) - return; - // empty response from server - if (line.isEmpty()) - throw new UnknownError("Received empty response from server!"); - // assertions - if ((assertion & ASSERT_BOT) == ASSERT_BOT && line.contains("error code=\"assertbotfailed\"")) - // assert !line.contains("error code=\"assertbotfailed\"") : "Bot privileges missing or revoked, or session expired."; - throw new AssertionError("Bot privileges missing or revoked, or session expired."); - if ((assertion & ASSERT_USER) == ASSERT_USER && line.contains("error code=\"assertuserfailed\"")) - // assert !line.contains("error code=\"assertuserfailed\"") : "Session expired."; - throw new AssertionError("Session expired."); - // protected page errors - if (line.matches("(protectednamespace|customcssjsprotected|cascadeprotected|protectedpage|protectedtitle)")) - throw new CredentialNotFoundException("Page is protected."); - if (line.contains("error code=\"permissiondenied\"")) - throw new CredentialNotFoundException("Permission denied."); // session expired or stupidity - // blocked! (note here the \" in blocked is deliberately missing for emailUser() - if (line.contains("error code=\"blocked") || line.contains("error code=\"autoblocked\"")) - { - log(Level.SEVERE, caller, "Cannot " + caller + " - user is blocked!."); - throw new AccountLockedException("Current user is blocked!"); - } - // unknown error - if (line.contains("error code=\"unknownerror\"")) - throw new UnknownError("Unknown MediaWiki API error, response was " + line); - // generic (automatic retry) - throw new IOException("MediaWiki error, response was " + line); + if (!line.contains(" constructRevisionString(long[] ids) { // sort and remove duplicates per https://mediawiki.org/wiki/API String[] sortedids = Arrays.stream(ids) .distinct() + .filter(id -> id >= 0) .sorted() .mapToObj(String::valueOf) .toArray(String[]::new); - + StringBuilder buffer = new StringBuilder(); - ArrayList chunks = new ArrayList<>(); + List chunks = new ArrayList<>(); for (int i = 0; i < sortedids.length; i++) { buffer.append(sortedids[i]); - if (i == ids.length - 1 || i == slowmax - 1) + if (i == ids.length - 1 || (i % slowmax) == slowmax - 1) { chunks.add(buffer.toString()); buffer.setLength(0); } else - buffer.append("%7C"); + buffer.append('|'); } - return chunks.toArray(new String[chunks.size()]); + return chunks; } /** * Cuts up a list of titles into batches for prop=X&titles=Y type queries. - * @param lengthBaseUrl the length of the url when no titles are present * @param titles a list of titles. - * @param limit whether to apply the maximum URL size * @return the titles ready for insertion into a URL - * @throws IOException if a network error occurs * @since 0.29 */ - protected String[] constructTitleString(int lengthBaseUrl, String[] titles, boolean limit) throws IOException + protected List constructTitleString(List titles) { - // sort and remove duplicates per https://mediawiki.org/wiki/API - Set set = new TreeSet<>(); - for (String title : titles) - set.add(encode(title, true)); - String[] titlesEnc = set.toArray(new String[set.size()]); - + // sort and remove duplicates + List titles_unique = titles.stream().sorted().distinct().collect(Collectors.toList()); + // actually construct the string - String titleStringToken = encode("|", false); ArrayList ret = new ArrayList<>(); - StringBuilder buffer = new StringBuilder(); - buffer.append(titlesEnc[0]); - int num = 1; - for (int i = num; i < titlesEnc.length; i++) + for (int i = 0; i < titles_unique.size() / slowmax + 1; i++) { - if (num < slowmax && - buffer.length() + titleStringToken.length() + titlesEnc[i].length() < URL_LENGTH_LIMIT - lengthBaseUrl) - { - buffer.append(titleStringToken); - } - else - { - ret.add(buffer.toString()); - buffer.setLength(0); - num = 0; - } - buffer.append(titlesEnc[i]); - ++num; + ret.add(String.join("|", + titles_unique.subList(i * slowmax, Math.min(titles_unique.size(), (i + 1) * slowmax)))); } - ret.add(buffer.toString()); - return ret.toArray(new String[ret.size()]); - } - - /** - * UTF-8 encode the String with URLEncoder after optional normalization; - * Usually, normalize should be set to true when a title or name String is - * passed in as an argument of a method. - * - * @param text the text to encode - * @param normalize if the text should be normalized first - * @return the encoded text - * @throws IOException if a network error occurs during initialization of the namespaces - */ - private String encode(String text, boolean normalize) throws IOException - { - final String encoding = "UTF-8"; - if (normalize) - text = normalize(text); - return URLEncoder.encode(text, encoding); + return ret; } /** * Convenience method for normalizing MediaWiki titles. (Converts all * underscores to spaces, localizes namespace names, fixes case of first - * char and does some other unicode fixes). + * char and does some other unicode fixes). Beware that this will not + * produce the same results as server-side normalization in a few corner + * cases, most notably: HTML entities and gender distinction in the user + * namespace prefix. * @param s the string to normalize * @return the normalized string * @throws IllegalArgumentException if the title is invalid - * @throws IOException if a network error occurs + * @throws UncheckedIOException if the namespace cache has not been + * populated, and a network error occurs when populating it * @since 0.27 */ - public String normalize(String s) throws IOException + public String normalize(String s) { + // remove section names + if (s.contains("#")) + s = s.substring(0, s.indexOf("#")); // remove leading colon if (s.startsWith(":")) s = s.substring(1); + s = s.replace('_', ' ').trim(); if (s.isEmpty()) - return s; + throw new IllegalArgumentException("Empty or whitespace only title."); int ns = namespace(s); // localize namespace names if (ns != MAIN_NAMESPACE) { - int colon = s.indexOf(":"); + int colon = s.indexOf(':'); s = namespaceIdentifier(ns) + s.substring(colon); } char[] temp = s.toCharArray(); @@ -7583,7 +8600,7 @@ public String normalize(String s) throws IOException { switch (temp[i]) { - // illegal characters + // illegal characters case '{': case '}': case '<': @@ -7592,13 +8609,10 @@ public String normalize(String s) throws IOException case ']': case '|': throw new IllegalArgumentException(s + " is an illegal title"); - case '_': - temp[i] = ' '; - break; } } // https://mediawiki.org/wiki/Unicode_normalization_considerations - String temp2 = new String(temp).trim().replaceAll("\\s+", " "); + String temp2 = new String(temp).replaceAll("\\s+", " "); return Normalizer.normalize(temp2, Normalizer.Form.NFC); } @@ -7607,7 +8621,7 @@ public String normalize(String s) throws IOException * and other write actions. * @since 0.30 */ - private synchronized void throttle() + protected synchronized void throttle() { try { @@ -7627,7 +8641,7 @@ private synchronized void throttle() * Checks whether the currently logged on user has sufficient rights to * edit/move a protected page. * - * @param pageinfo the output from {@link #getPageInfo(java.lang.String)} + * @param pageinfo the output from {@link #getPageInfo} for an article * @param action what we are doing * @return whether the user can perform the specified action * @throws IOException if a network error occurs @@ -7644,57 +8658,11 @@ protected boolean checkRights(Map pageinfo, String action) throw if (level.equals(FULL_PROTECTION)) return user.isAllowedTo("editprotected"); } - if ((Boolean)protectionstate.get("cascade") == Boolean.TRUE) // can be null + if (Boolean.TRUE.equals(protectionstate.get("cascade"))) return user.isAllowedTo("editprotected"); return true; } - // cookie methods - - /** - * Sets cookies to an unconnected URLConnection and enables gzip - * compression of returned text. - * @param u an unconnected URLConnection - */ - protected void setCookies(URLConnection u) - { - StringBuilder cookie = new StringBuilder(100); - for (Map.Entry entry : cookies.entrySet()) - { - cookie.append(entry.getKey()); - cookie.append("="); - cookie.append(entry.getValue()); - cookie.append("; "); - } - u.setRequestProperty("Cookie", cookie.toString()); - - // enable gzip compression - if (zipped) - u.setRequestProperty("Accept-encoding", "gzip"); - u.setRequestProperty("User-Agent", useragent); - } - - /** - * Grabs cookies from the URL connection provided. - * @param u an unconnected URLConnection - */ - private void grabCookies(URLConnection u) - { - String headerName; - for (int i = 1; (headerName = u.getHeaderFieldKey(i)) != null; i++) - if (headerName.equalsIgnoreCase("Set-Cookie")) - { - String cookie = u.getHeaderField(i); - cookie = cookie.substring(0, cookie.indexOf(';')); - String name = cookie.substring(0, cookie.indexOf('=')); - String value = cookie.substring(cookie.indexOf('=') + 1, cookie.length()); - // these cookies were pruned, but are still sent for some reason? - // TODO: when these cookies are no longer sent, remove this test - if (!value.equals("deleted")) - cookies.put(name, value); - } - } - // logging methods /** @@ -7719,30 +8687,4 @@ protected void logurl(String url, String method) { logger.logp(Level.INFO, "Wiki", method, "Fetching URL {0}", url); } - - // serialization - - /** - * Writes this wiki to a file. - * @param out an ObjectOutputStream to write to - * @throws IOException if there are local IO problems - * @since 0.10 - */ - private void writeObject(ObjectOutputStream out) throws IOException - { - out.defaultWriteObject(); - } - - /** - * Reads a copy of a wiki from a file. - * @param in an ObjectInputStream to read from - * @throws IOException if there are local IO problems - * @throws ClassNotFoundException if we can't recognize the input - * @since 0.10 - */ - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException - { - in.defaultReadObject(); - statuscounter = statusinterval; // force a status check on next edit - } } diff --git a/src/pattypan/Main.java b/src/pattypan/Main.java index f44ce93..4724a19 100644 --- a/src/pattypan/Main.java +++ b/src/pattypan/Main.java @@ -84,7 +84,7 @@ public static void main(String[] args) { new String[]{os, Settings.VERSION} ); - Session.WIKI = new Wiki(wiki, scriptPath, protocol); + Session.WIKI = Wiki.newSession(wiki, scriptPath, protocol); launch(args); } } diff --git a/src/pattypan/Session.java b/src/pattypan/Session.java index 6ae27a4..073d65e 100644 --- a/src/pattypan/Session.java +++ b/src/pattypan/Session.java @@ -51,7 +51,7 @@ public final class Session { public static String WIKICODE = ""; public static ArrayList VARIABLES = new ArrayList<>(Arrays.asList("path", "name")); - public static Wiki WIKI = new Wiki("commons.wikimedia.org"); + public static Wiki WIKI = Wiki.newSession("commons.wikimedia.org"); public static ArrayList FILES_TO_UPLOAD = new ArrayList<>(); static { From 14f58250ffcda0cb0e1d995659fe94044a4cc6a7 Mon Sep 17 00:00:00 2001 From: Albin Larsson Date: Sat, 9 Oct 2021 11:55:35 +0200 Subject: [PATCH 02/18] bump version --- src/pattypan/Settings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pattypan/Settings.java b/src/pattypan/Settings.java index d39f08a..10882c2 100644 --- a/src/pattypan/Settings.java +++ b/src/pattypan/Settings.java @@ -42,7 +42,7 @@ public final class Settings { private Settings() {}; public static final String NAME = "pattypan"; - public static final String VERSION = "20.04"; + public static final String VERSION = "21.10"; public static final String USERAGENT = NAME + "/" + VERSION + " (https://github.com/yarl/pattypan)"; public static final Map SETTINGS = new HashMap<>(); From 39e4b6a3d9dbd5a252be6dccdf0f52e240d0261d Mon Sep 17 00:00:00 2001 From: Albin Larsson Date: Sat, 9 Oct 2021 12:18:47 +0200 Subject: [PATCH 03/18] Revert "Update wiki-java #138" --- src/org/wikipedia/Wiki.java | 8002 +++++++++++++++-------------------- src/pattypan/Main.java | 2 +- src/pattypan/Session.java | 2 +- 3 files changed, 3532 insertions(+), 4474 deletions(-) diff --git a/src/org/wikipedia/Wiki.java b/src/org/wikipedia/Wiki.java index 14aa8d2..c378f0f 100644 --- a/src/org/wikipedia/Wiki.java +++ b/src/org/wikipedia/Wiki.java @@ -1,6 +1,6 @@ /** - * @(#)Wiki.java 0.36 08/02/2019 - * Copyright (C) 2007 - 2019 MER-C and contributors + * @(#)Wiki.java 0.33 10/08/2017 + * Copyright (C) 2007 - 2017 MER-C and contributors * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License @@ -22,65 +22,45 @@ import java.io.*; import java.net.*; -import java.net.http.*; import java.nio.*; -import java.nio.charset.StandardCharsets; import java.nio.file.*; import java.text.Normalizer; import java.time.*; import java.time.format.*; -import java.time.temporal.ChronoUnit; import java.util.*; -import java.util.concurrent.*; import java.util.function.*; import java.util.logging.*; -import java.util.stream.*; +import java.util.stream.Collectors; import java.util.zip.GZIPInputStream; import javax.security.auth.login.*; /** * This is a somewhat sketchy bot framework for editing MediaWiki wikis. - * Requires JDK 11 or greater. Uses the MediaWiki API for most * operations. It is recommended that the server runs the latest version - * of MediaWiki (1.31), otherwise some functions may not work. This framework - * requires no dependencies outside the core JDK and does not implement any - * functionality added by MediaWiki extensions. + * of MediaWiki (1.31), otherwise some functions may not work. *

* Extended documentation is available * here. * All wikilinks are relative to the English Wikipedia and all timestamps are in * your wiki's time zone. - *

+ *

* Please file bug reports here - * or at the Github issue - * tracker. - * - *

Configuration variables

- *

- * Some configuration is available through java.util.Properties. - * Set the system property wiki-java.properties to a file path - * where a configuration file is located. The available variables are: - *

    - *
  • maxretries: (default 2) the number of attempts to retry a network - * request before stopping - *
  • connecttimeout: (default 30000) maximum allowed time for a HTTP(s) - * connection to be established in milliseconds - *
  • readtimeout: (default 180000) maximum allowed time for the read - * to take place in milliseconds (needs to be longer, some connections are - * slow and the data volume is large!). - *
  • loguploadsize: (default 22, equivalent to 2^22 = 4 MB) controls - * the log2(size) of each chunk in chunked uploads. Disable chunked uploads - * by setting a large value here (50, equivalent to 2^50 = 1 PB will do). - * Stuff you actually upload must be no larger than 4 GB. - * .
+ * or at the Github issue + * tracker. * * @author MER-C and contributors - * @version 0.36 + * @version 0.33 */ -public class Wiki implements Comparable +public class Wiki implements Serializable { + // Master TODO list: + // *Admin stuff + // *More multiqueries + // *Generators (hard) + // NAMESPACES /** @@ -284,12 +264,6 @@ public class Wiki implements Comparable */ public static final String PATROL_LOG = "patrol"; - /** - * Denotes the page creation log. - * @since 0.36 - */ - public static final String PAGE_CREATION_LOG = "create"; - // PROTECTION LEVELS /** @@ -299,14 +273,14 @@ public class Wiki implements Comparable public static final String NO_PROTECTION = "all"; /** - * Denotes semi-protection (only autoconfirmed users can perform a - * action). + * Denotes semi-protection (i.e. only autoconfirmed users can perform a + * particular action). * @since 0.09 */ public static final String SEMI_PROTECTION = "autoconfirmed"; /** - * Denotes full protection (only admins can perfom a particular action). + * Denotes full protection (i.e. only admins can perfom a particular action). * @since 0.09 */ public static final String FULL_PROTECTION = "sysop"; @@ -314,21 +288,21 @@ public class Wiki implements Comparable // ASSERTION MODES /** - * Use no assertions. + * Use no assertions (i.e. 0). * @see #setAssertionMode * @since 0.11 */ public static final int ASSERT_NONE = 0; /** - * Assert that we are logged in. This is checked every action. + * Assert that we are logged in (i.e. 1). This is checked every action. * @see #setAssertionMode * @since 0.30 */ public static final int ASSERT_USER = 1; /** - * Assert that we have a bot flag. This is checked every action. + * Assert that we have a bot flag (i.e. 2). This is checked every action. * @see #setAssertionMode * @since 0.11 */ @@ -343,41 +317,74 @@ public class Wiki implements Comparable public static final int ASSERT_NO_MESSAGES = 4; /** - * Assert that we have a sysop flag. This is checked intermittently. + * Assert that we have a sysop flag (i.e. 8). This is checked intermittently. * @see #setAssertionMode * @since 0.30 */ public static final int ASSERT_SYSOP = 8; + // RC OPTIONS + + /** + * In queries against the recent changes table, this would mean we don't + * fetch anonymous edits. + * @since 0.20 + */ + public static final int HIDE_ANON = 1; + + /** + * In queries against the recent changes table, this would mean we don't + * fetch edits made by bots. + * @since 0.20 + */ + public static final int HIDE_BOT = 2; + + /** + * In queries against the recent changes table, this would mean we don't + * fetch by the logged in user. + * @since 0.20 + */ + public static final int HIDE_SELF = 4; + + /** + * In queries against the recent changes table, this would mean we don't + * fetch minor edits. + * @since 0.20 + */ + public static final int HIDE_MINOR = 8; + + /** + * In queries against the recent changes table, this would mean we don't + * fetch patrolled edits. + * @since 0.20 + */ + public static final int HIDE_PATROLLED = 16; + // REVISION OPTIONS /** - * In {@link org.wikipedia.Wiki.Revision#diff(long) Revision.diff()}, - * denotes the next revision. - * @see org.wikipedia.Wiki.Revision#diff(long) + * In Revision.diff(), denotes the next revision. + * @see org.wikipedia.Wiki.Revision#diff(org.wikipedia.Wiki.Revision) * @since 0.21 */ public static final long NEXT_REVISION = -1L; /** - * In {@link org.wikipedia.Wiki.Revision#diff(long) Revision.diff()}, - * denotes the current revision. - * @see org.wikipedia.Wiki.Revision#diff(long) + * In Revision.diff(), denotes the current revision. + * @see org.wikipedia.Wiki.Revision#diff(org.wikipedia.Wiki.Revision) * @since 0.21 */ public static final long CURRENT_REVISION = -2L; /** - * In {@link org.wikipedia.Wiki.Revision#diff(long) Revision.diff()}, - * denotes the previous revision. - * @see org.wikipedia.Wiki.Revision#diff(long) + * In Revision.diff(), denotes the previous revision. + * @see org.wikipedia.Wiki.Revision#diff(org.wikipedia.Wiki.Revision) * @since 0.21 */ public static final long PREVIOUS_REVISION = -3L; /** * The list of options the user can specify for his/her gender. - * @see User#getGender() * @since 0.24 */ public enum Gender @@ -405,290 +412,230 @@ public enum Gender unknown; } - private static final String version = "0.36"; - - // fundamental URL strings - private final String protocol, domain, scriptPath; - private String base, articleUrl; - - /** - * Stores default HTTP parameters for API calls. Contains {@linkplain - * #setMaxLag(int) maxlag}, {@linkplain #setResolveRedirects(boolean) redirect - * resolution} and {@linkplain #setAssertionMode(int) user and bot assertions} - * when wanted by default. Add stuff to this map if you want to add parameters - * to every API call. - * @see #makeApiCall(Map, Map, String) - */ - protected ConcurrentHashMap defaultApiParams; - - /** - * URL entrypoint for the MediaWiki API. (Needs to be accessible to - * subclasses.) - * @see #initVars() - * @see #getApiUrl() - * @see MediaWiki - * documentation - */ - protected String apiUrl; + private static final String version = "0.33"; - // wiki properties - private boolean siteinfofetched = false; + // the domain of the wiki + private String domain; + protected String query, base, apiUrl; + protected String scriptPath = "/w"; private boolean wgCapitalLinks = true; - private String dbname; - private String mwVersion; - private ZoneId timezone = ZoneOffset.UTC; - private Locale locale = Locale.ENGLISH; - private List extensions = Collections.emptyList(); - private LinkedHashMap namespaces = null; - private ArrayList ns_subpages = null; + private ZoneId timezone = ZoneId.of("UTC"); // user management - private HttpClient client; - private final CookieManager cookies; + private Map cookies = new HashMap<>(12); private User user; private int statuscounter = 0; - // watchlist cache - private List watchlist = null; + // various caches + private transient LinkedHashMap namespaces = null; + private transient ArrayList ns_subpages = null; + private transient List watchlist = null; // preferences private int max = 500; private int slowmax = 50; - private int throttle = 10000; + private int throttle = 10000; // throttle private int maxlag = 5; private int assertion = ASSERT_NONE; // assertion mode - private int statusinterval = 100; // status check + private transient int statusinterval = 100; // status check private int querylimit = Integer.MAX_VALUE; private String useragent = "Wiki.java/" + version + " (https://github.com/MER-C/wiki-java/)"; private boolean zipped = true; private boolean markminor = false, markbot = false; private boolean resolveredirect = false; + private String protocol = "https://"; private Level loglevel = Level.ALL; private static final Logger logger = Logger.getLogger("wiki"); // Store time when the last throttled action was executed private long lastThrottleActionTime = 0; - // config via properties - private final int maxtries; - private final int read_timeout_msec; - private final int log2_upload_size; + // retry count + private int maxtries = 2; + + // serial version + private static final long serialVersionUID = -8745212681497643456L; + + // time to open a connection + private static final int CONNECTION_CONNECT_TIMEOUT_MSEC = 30000; // 30 seconds + // time for the read to take place. (needs to be longer, some connections are slow + // and the data volume is large!) + private static final int CONNECTION_READ_TIMEOUT_MSEC = 180000; // 180 seconds + // log2(upload chunk size). Default = 22 => upload size = 4 MB. Disable + // chunked uploads by setting a large value here (50 = 1 PB will do). + private static final int LOG2_CHUNK_SIZE = 22; + // maximum URL length in bytes + protected static final int URL_LENGTH_LIMIT = 8192; // CONSTRUCTORS AND CONFIGURATION /** - * Creates a new MediaWiki API client for the given wiki with - * $wgScriptPath set to scriptPath and via the - * specified protocol. + * Creates a new connection to the English Wikipedia via HTTPS. + * @since 0.02 + * @deprecated use Wiki#createInstance instead + */ + @Deprecated + public Wiki() + { + this("en.wikipedia.org", "/w"); + } + + /** + * Creates a new connection to a wiki via HTTPS. WARNING: if the wiki uses + * a $wgScriptpath other than the default /w, you need to call + * getScriptPath() to automatically set it. Alternatively, you + * can use the constructor below if you know it in advance. + * + * @param domain the wiki domain name e.g. en.wikipedia.org (defaults to + * en.wikipedia.org) + * @deprecated use Wiki#createInstance instead + */ + @Deprecated + public Wiki(String domain) + { + this(domain, "/w"); + } + + /** + * Creates a new connection to a wiki with $wgScriptpath set to + * scriptPath via HTTPS. * * @param domain the wiki domain name * @param scriptPath the script path - * @param protocol a protocol e.g. "http://", "https://" or "file:///" - * @since 0.31 + * @since 0.14 + * @deprecated use Wiki#createInstance instead */ - protected Wiki(String domain, String scriptPath, String protocol) + @Deprecated + public Wiki(String domain, String scriptPath) { - this.domain = Objects.requireNonNull(domain); - this.scriptPath = Objects.requireNonNull(scriptPath); - this.protocol = Objects.requireNonNull(protocol); - - defaultApiParams = new ConcurrentHashMap<>(); - defaultApiParams.put("format", "xml"); - defaultApiParams.put("maxlag", String.valueOf(maxlag)); + this(domain, scriptPath, "https://"); + } + /** + * Creates a new connection to a wiki with $wgScriptpath set to + * scriptPath via the specified protocol. + * + * @param domain the wiki domain name + * @param scriptPath the script path + * @param protocol a protocol e.g. "http://", "https://" or "file:///" + * @since 0.31 + * @deprecated use Wiki#createInstance instead; this will be made private + * for the reason in the TODO comment below + */ + @Deprecated + public Wiki(String domain, String scriptPath, String protocol) + { + if (domain == null || domain.isEmpty()) + domain = "en.wikipedia.org"; + this.domain = domain; + this.scriptPath = scriptPath; + this.protocol = protocol; + + // init variables + // This is fine as long as you do not have parameters other than domain + // and scriptpath in constructors and do not do anything else than super(x)! + // http://stackoverflow.com/questions/3404301/whats-wrong-with-overridable-method-calls-in-constructors + // TODO: remove this logger.setLevel(loglevel); logger.log(Level.CONFIG, "[{0}] Using Wiki.java {1}", new Object[] { domain, version }); - - // read in config - Properties props = new Properties(); - String filename = System.getProperty("wiki-java.properties"); - if (filename != null) - { - try - { - InputStream in = new FileInputStream(new File(filename)); - props.load(in); - } - catch (IOException ex) - { - logger.log(Level.WARNING, "Unable to load properties file " + filename); - } - } - maxtries = Integer.parseInt(props.getProperty("maxretries", "2")); - log2_upload_size = Integer.parseInt(props.getProperty("loguploadsize", "22")); // 4 MB - read_timeout_msec = Integer.parseInt(props.getProperty("readtimeout", "180000")); // 180 seconds - cookies = new CookieManager(null, CookiePolicy.ACCEPT_ALL); - client = HttpClient.newBuilder() - .connectTimeout(Duration.ofSeconds(30)) - .cookieHandler(cookies) - .build(); + initVars(); } - + /** - * Creates a new MediaWiki API client for the given wiki using HTTPS. - * Depending on the settings of the wiki, you may need to call {@link - * Wiki#getSiteInfo()} on the returned object after this in order for some - * functionality to work correctly. + * Creates a new connection to a wiki via HTTPS. Depending on the settings + * of the wiki, you may need to call {@link Wiki#getSiteInfo()} on the + * returned object after this in order for some functionality to work + * correctly. * * @param domain the wiki domain name e.g. en.wikipedia.org (defaults to * en.wikipedia.org) - * @return the constructed API client object + * @return the created wiki * @since 0.34 */ - public static Wiki newSession(String domain) + public static Wiki createInstance(String domain) { - return newSession(domain, "/w", "https://"); + return createInstance(domain, "/w", "https://"); } - + /** - * Creates a new MediaWiki API client for the given wiki with - * $wgScriptPath set to scriptPath and via the - * specified protocol. Depending on the settings of the wiki, you may need - * to call {@link Wiki#getSiteInfo()} on the returned object after this in + * $wgScriptPath set to scriptPath and via the + * specified protocol. Depending on the settings of the wiki, you may need + * to call {@link Wiki#getSiteInfo()} on the returned object after this in * order for some functionality to work correctly. * - *

All factory methods in subclasses must call {@link #initVars()}. - * * @param domain the wiki domain name * @param scriptPath the script path * @param protocol a protocol e.g. "http://", "https://" or "file:///" - * @return the constructed API client object + * @return the constructed Wiki object * @since 0.34 */ - public static Wiki newSession(String domain, String scriptPath, String protocol) + public static Wiki createInstance(String domain, String scriptPath, String protocol) { // Don't put network requests here. Servlets cannot afford to make // unnecessary network requests in initialization. - Wiki wiki = new Wiki(domain, scriptPath, protocol); - wiki.initVars(); + Wiki wiki = new Wiki(domain, "/w", protocol); + wiki.initVars(); // construct URL bases return wiki; } /** * Edit this if you need to change the API and human interface url - * configuration of the wiki. One example use is to change the port number. + * configuration of the wiki. One example use is server-side cache + * management (maxage and smaxage API parameters). * *

Contributed by Tedder * @since 0.24 */ protected void initVars() { - base = protocol + domain + scriptPath + "/index.php"; - apiUrl = protocol + domain + scriptPath + "/api.php"; - articleUrl = protocol + domain + "/wiki/"; + StringBuilder basegen = new StringBuilder(protocol); + basegen.append(domain); + basegen.append(scriptPath); + StringBuilder apigen = new StringBuilder(basegen); + apigen.append("/api.php?format=xml&"); + // MediaWiki has inbuilt maxlag functionality, see [[mw:Manual:Maxlag + // parameter]]. Let's exploit it. + if (maxlag >= 0) + { + apigen.append("maxlag="); + apigen.append(maxlag); + apigen.append("&"); + basegen.append("/index.php?maxlag="); + basegen.append(maxlag); + basegen.append("&title="); + } + else + basegen.append("/index.php?title="); + base = basegen.toString(); + // the native API supports assertions as of MW 1.23 + if ((assertion & ASSERT_BOT) == ASSERT_BOT) + apigen.append("assert=bot&"); + else if ((assertion & ASSERT_USER) == ASSERT_USER) + apigen.append("assert=user&"); + apiUrl = apigen.toString(); + apigen.append("action=query&"); + if (resolveredirect) + apigen.append("redirects&"); + query = apigen.toString(); } /** - * Gets the domain of the wiki as supplied on construction. + * Gets the domain of the wiki, as supplied on construction. * @return the domain of the wiki * @since 0.06 */ - public final String getDomain() + public String getDomain() { return domain; } - - /** - * Gets the - * $wgScriptPath variable as supplied on construction. - * @return the script path of the wiki - * @since 0.14 - */ - public final String getScriptPath() - { - return scriptPath; - } - - /** - * Gets the protocol used to access this MediaWiki instance, as supplied - * on construction. - * @return (see above) - * @since 0.35 - */ - public final String getProtocol() + + public String getProtocol() { return protocol; } - /** - * Determines whether this wiki is equal to another object based on the - * protocol (not case sensitive), domain (not case sensitive) and - * scriptPath (case sensitive). A return value of {@code true} means two - * Wiki objects point to the same instance of MediaWiki. - * @param obj the object to compare - * @return whether the wikis point to the same instance of MediaWiki - */ - @Override - public boolean equals(Object obj) - { - if (!(obj instanceof Wiki)) - return false; - Wiki other = (Wiki)obj; - return domain.equalsIgnoreCase(other.domain) - && scriptPath.equals(other.scriptPath) - && protocol.equalsIgnoreCase(other.protocol); - } - - /** - * Returns a hash code of this object based on the protocol, domain and - * scriptpath. - * @return a hash code - */ - @Override - public int hashCode() - { - // English locale used here for reproducability and so network requests - // are not required - int hc = domain.toLowerCase(Locale.ENGLISH).hashCode(); - hc = 127 * hc + scriptPath.hashCode(); - hc = 127 * hc + protocol.toLowerCase(Locale.ENGLISH).hashCode(); - return hc; - } - - /** - * Allows wikis to be sorted based on their domain (case insensitive), then - * their script path (case sensitive). If 0 is returned, it is reasonable - * both Wikis point to the same instance of MediaWiki. - * @param other the wiki to compare to - * @return -1 if this wiki is alphabetically before the other, 1 if after - * and 0 if they are likely to be the same instance of MediaWiki - * @since 0.35 - */ - @Override - public int compareTo(Wiki other) - { - int result = domain.compareToIgnoreCase(other.domain); - if (result == 0) - result = scriptPath.compareTo(other.scriptPath); - return result; - } - - /** - * Gets the URL of index.php. - * @return (see above) - * @see - * MediaWiki documentation - * @since 0.35 - */ - public String getIndexPhpUrl() - { - return base; - } - - /** - * Gets the URL of api.php. - * @return (see above) - * @see MediaWiki - * documentation - * @since 0.36 - */ - public String getApiUrl() - { - return apiUrl; - } - /** * Gets the editing throttle. * @return the throttle value in milliseconds @@ -701,11 +648,8 @@ public int getThrottle() } /** - * Sets the throttle, which limits most write requests to no more than one - * per wiki instance in the given time across all threads. (As a - * consequence, all throttled methods are thread safe.) Read requests are - * not throttled or restricted in any way. Default is 10 seconds. - * + * Sets the editing throttle. Read requests are not throttled or restricted + * in any way. Default is 10s. * @param throttle the new throttle value in milliseconds * @see #getThrottle * @since 0.09 @@ -718,166 +662,69 @@ public void setThrottle(int throttle) /** * Gets various properties of the wiki and sets the bot framework up to use - * them. The return value is cached. This method is thread safe. Returns: + * them. Also populates the namespace cache. Returns: *

    *
  • usingcapitallinks: (Boolean) whether a wiki forces upper case * for the title. Example: en.wikipedia = true, en.wiktionary = false. * Default = true. See * $wgCapitalLinks - *
  • scriptpath: (String) the scriptpath: (String) the - * $wgScriptPath wiki variable. Default = {@code /w}. + * $wgScriptPath wiki variable. Default = /w. *
  • version: (String) the MediaWiki version used for this wiki - *
  • timezone: (ZoneId) the timezone the wiki is in, default = UTC - *
  • locale: (Locale) the locale of the wiki - *
  • dbname: (String) the internal name of the database + *
  • timezone: (String) the timezone the wiki is in, default = UTC *
* * @return (see above) * @since 0.30 * @throws IOException if a network error occurs - * @deprecated This method is likely going to get renamed with the return - * type changed to void once I finish cleaning up the site info caching - * mechanism. Use the specialized methods instead. - */ - @Deprecated - public synchronized Map getSiteInfo() throws IOException - { - Map siteinfo = new HashMap<>(); - if (!siteinfofetched) - { - Map getparams = new HashMap<>(); - getparams.put("action", "query"); - getparams.put("meta", "siteinfo"); - getparams.put("siprop", "namespaces|namespacealiases|general|extensions"); - String line = makeApiCall(getparams, null, "getSiteInfo"); - detectUncheckedErrors(line, null, null); - - // general site info - String bits = line.substring(line.indexOf("")); - wgCapitalLinks = parseAttribute(bits, "case", 0).equals("first-letter"); - timezone = ZoneId.of(parseAttribute(bits, "timezone", 0)); - mwVersion = parseAttribute(bits, "generator", 0); - locale = new Locale(parseAttribute(bits, "lang", 0)); - dbname = parseAttribute(bits, "wikiid", 0); - - // parse extensions - bits = line.substring(line.indexOf(""), line.indexOf("")); - extensions = new ArrayList<>(); - String[] unparsed = bits.split("(30); - ns_subpages = new ArrayList<>(30); - // xml form: Media or - String[] items = line.split("') + 1; - int c = items[i].indexOf(""); - if (c < 0) - namespaces.put("", ns); - else - namespaces.put(normalize(decode(items[i].substring(b, c))), ns); - - String canonicalnamespace = parseAttribute(items[i], "canonical", 0); - if (canonicalnamespace != null) - namespaces.put(canonicalnamespace, ns); - - // does this namespace support subpages? - if (items[i].contains("subpages=\"\"")) - ns_subpages.add(ns); - } - siteinfofetched = true; - log(Level.INFO, "getSiteInfo", "Successfully retrieved site info for " + getDomain()); - } - siteinfo.put("usingcapitallinks", wgCapitalLinks); - siteinfo.put("scriptpath", scriptPath); - siteinfo.put("timezone", timezone); - siteinfo.put("version", mwVersion); - siteinfo.put("locale", locale); - siteinfo.put("extensions", extensions); - siteinfo.put("dbname", dbname); - return siteinfo; - } - - /** - * Gets the version of MediaWiki this wiki runs e.g. 1.20wmf5 (54b4fcb). - * See [[Special:Version]] on your wiki. - * @return (see above) - * @throws UncheckedIOException if the site info cache has not been - * populated and a network error occurred when populating it - * @since 0.14 - * @see MediaWiki Git - */ - public String version() - { - ensureNamespaceCache(); - return mwVersion; - } - - /** - * Detects whether a wiki forces upper case for the first character in a - * title. Example: en.wikipedia = true, en.wiktionary = false. - * @return (see above) - * @throws UncheckedIOException if the site info cache has not been - * populated and a network error occurred when populating it - * @see MediaWiki - * documentation - * @since 0.30 - */ - public boolean usesCapitalLinks() - { - ensureNamespaceCache(); - return wgCapitalLinks; - } - - /** - * Returns the list of extensions installed on this wiki. - * @return (see above) - * @throws UncheckedIOException if the site info cache has not been - * populated and a network error occurred when populating it - * @see MediaWiki - * documentation - * @since 0.35 - */ - public List installedExtensions() - { - ensureNamespaceCache(); - return new ArrayList<>(extensions); - } - - /** - * Gets the timezone of this wiki - * @return (see above) - * @throws UncheckedIOException if the site info cache has not been - * populated and a network error occurred when populating it - * @since 0.35 - */ - public ZoneId timezone() - { - ensureNamespaceCache(); - return timezone; - } - - /** - * Gets the locale of this wiki. - * @return (see above) - * @throws UncheckedIOException if the site info cache has not been - * populated and a network error occurred when populating it - * @since 0.35 */ - public Locale locale() + public Map getSiteInfo() throws IOException { - ensureNamespaceCache(); - return locale; + Map ret = new HashMap<>(); + String line = fetch(query + "action=query&meta=siteinfo&siprop=namespaces%7Cnamespacealiases%7Cgeneral", "getSiteInfo"); + + // general site info + String bits = line.substring(line.indexOf("")); + wgCapitalLinks = parseAttribute(bits, "case", 0).equals("first-letter"); + ret.put("usingcapitallinks", wgCapitalLinks); + scriptPath = parseAttribute(bits, "scriptpath", 0); + ret.put("scriptpath", scriptPath); + timezone = ZoneId.of(parseAttribute(bits, "timezone", 0)); + ret.put("timezone", timezone); + ret.put("version", parseAttribute(bits, "generator", 0)); + + // populate namespace cache + namespaces = new LinkedHashMap<>(30); + ns_subpages = new ArrayList<>(30); + // xml form: Media or + String[] items = line.split("') + 1; + int c = items[i].indexOf(""); + if (c < 0) + namespaces.put("", ns); + else + namespaces.put(normalize(decode(items[i].substring(b, c))), ns); + + String canonicalnamespace = parseAttribute(items[i], "canonical", 0); + if (canonicalnamespace != null) + namespaces.put(canonicalnamespace, ns); + + // does this namespace support subpages? + if (items[i].contains("subpages=\"\"")) + ns_subpages.add(ns); + } + + initVars(); + log(Level.INFO, "getSiteInfo", "Successfully retrieved site info for " + getDomain()); + return ret; } /** @@ -906,9 +753,7 @@ public String getUserAgent() * Enables/disables GZip compression for GET requests. Default: true. * @param zipped whether we use GZip compression * @since 0.23 - * @deprecated this is now handled transparently; just delete calls to this method. */ - @Deprecated(forRemoval=true) public void setUsingCompressedRequests(boolean zipped) { this.zipped = zipped; @@ -919,9 +764,7 @@ public void setUsingCompressedRequests(boolean zipped) * Default: true. * @return (see above) * @since 0.23 - * @deprecated this is now handled transparently; just delete calls to this method. */ - @Deprecated(forRemoval=true) public boolean isUsingCompressedRequests() { return zipped; @@ -947,10 +790,7 @@ public boolean isResolvingRedirects() public void setResolveRedirects(boolean b) { resolveredirect = b; - if (b) - defaultApiParams.put("redirects", "1"); - else - defaultApiParams.remove("redirects"); + initVars(); } /** @@ -1023,6 +863,31 @@ public void setQueryLimit(int limit) querylimit = limit; } + /** + * Determines whether this wiki is equal to another object. + * @param obj the object to compare + * @return whether the domains of the wikis are equal + * @since 0.10 + */ + @Override + public boolean equals(Object obj) + { + if (!(obj instanceof Wiki)) + return false; + return domain.equals(((Wiki)obj).domain); + } + + /** + * Returns a hash code of this object. + * @return a hash code + * @since 0.12 + */ + @Override + public int hashCode() + { + return domain.hashCode() * maxlag - throttle; + } + /** * Returns a string representation of this Wiki. * @return a string representation of this Wiki. @@ -1032,14 +897,12 @@ public void setQueryLimit(int limit) public String toString() { // domain - StringBuilder buffer = new StringBuilder("Wiki[url="); - buffer.append(protocol); + StringBuilder buffer = new StringBuilder("Wiki[domain="); buffer.append(domain); - buffer.append(scriptPath); // user buffer.append(",user="); - buffer.append(Objects.toString(user)); + buffer.append(user != null ? user.toString() : "null"); buffer.append(","); // throttle mechanisms @@ -1085,10 +948,7 @@ public void setMaxLag(int lag) { maxlag = lag; log(Level.CONFIG, "setMaxLag", "Setting maximum allowable database lag to " + lag); - if (maxlag >= 0) - defaultApiParams.put("maxlag", String.valueOf(maxlag)); - else - defaultApiParams.remove("maxlag"); + initVars(); } /** @@ -1113,13 +973,7 @@ public void setAssertionMode(int mode) { assertion = mode; log(Level.CONFIG, "setAssertionMode", "Set assertion mode to " + mode); - - if ((assertion & ASSERT_BOT) == ASSERT_BOT) - defaultApiParams.put("assert", "bot"); - else if ((assertion & ASSERT_USER) == ASSERT_USER) - defaultApiParams.put("assert", "user"); - else - defaultApiParams.remove("assert"); + initVars(); } /** @@ -1167,36 +1021,29 @@ public void setLogLevel(Level loglevel) // META STUFF /** - * Logs in to the wiki. This method is thread-safe. + * Logs in to the wiki. This method is thread-safe. * * @param username a username - * @param password a password, as a {@code char[]} for security - * reasons. Overwritten once the password is used. + * @param password a password (as a char[] due to JPasswordField) * @throws IOException if a network error occurs - * @throws FailedLoginException if the login failed due to an incorrect - * username or password, the requirement for an interactive login (not - * supported, use [[Special:BotPasswords]]) or some other reason * @see #logout - * @see MediaWiki - * documentation */ public synchronized void login(String username, char[] password) throws IOException, FailedLoginException { - Map getparams = new HashMap<>(); - getparams.put("action", "login"); - Map postparams = new HashMap<>(); - postparams.put("lgname", username); - postparams.put("lgpassword", new String(password)); - postparams.put("lgtoken", getToken("login")); - String line = makeApiCall(getparams, postparams, "login"); - detectUncheckedErrors(line, null, null); - Arrays.fill(password, '0'); + StringBuilder buffer = new StringBuilder(500); + buffer.append("lgname="); + buffer.append(encode(username, false)); + buffer.append("&lgpassword="); + buffer.append(encode(new String(password), false)); + buffer.append("&lgtoken="); + buffer.append(encode(getToken("login"), false)); + String line = post(apiUrl + "action=login", buffer.toString(), "login"); + buffer.setLength(0); // check for success if (line.contains("result=\"Success\"")) { - String returned_username = parseAttribute(line, "lgusername", 0); - user = getUsers(List.of(returned_username)).get(0); + user = new User(parseAttribute(line, "lgusername", 0)); boolean apihighlimit = user.isAllowedTo("apihighlimits"); if (apihighlimit) { @@ -1208,21 +1055,18 @@ public synchronized void login(String username, char[] password) throws IOExcept else if (line.contains("result=\"Failed\"")) throw new FailedLoginException("Login failed: " + parseAttribute(line, "reason", 0)); // interactive login or bot password required - else if (line.contains("result=\"Aborted\"")) + else if (line.contains("result=\"Aborted\"")) throw new FailedLoginException("Login failed: you need to use a bot password, see [[Special:Botpasswords]]."); else throw new AssertionError("Unreachable!"); } /** - * Logs in to the wiki. This method is thread-safe. + * Logs in to the wiki. This method is thread-safe. * * @param username a username * @param password a string with the password * @throws IOException if a network error occurs - * @throws FailedLoginException if the login failed due to an incorrect - * username or password, the requirement for an interactive login (not - * supported, use [[Special:Botpasswords]]) or some other reason * @see #logout */ public synchronized void login(String username, String password) throws IOException, FailedLoginException @@ -1239,7 +1083,7 @@ public synchronized void login(String username, String password) throws IOExcept */ public synchronized void logout() { - cookies.getCookieStore().removeAll(); + cookies.clear(); user = null; max = 500; slowmax = 50; @@ -1258,15 +1102,10 @@ public synchronized void logout() * @since 0.14 * @see #login * @see #logout - * @see MediaWiki - * documentation */ public synchronized void logoutServerSide() throws IOException { - Map getparams = new HashMap<>(); - getparams.put("action", "logout"); - String response = makeApiCall(getparams, null, "logoutServerSide"); - detectUncheckedErrors(response, null, null); + fetch(apiUrl + "action=logout", "logoutServerSide"); logout(); // destroy local cookies } @@ -1279,18 +1118,12 @@ public synchronized void logoutServerSide() throws IOException */ public boolean hasNewMessages() throws IOException { - Map getparams = new HashMap<>(); - getparams.put("action", "query"); - getparams.put("meta", "userinfo"); - getparams.put("uiprop", "hasmsg"); - String response = makeApiCall(getparams, null, "hasNewMessages"); - detectUncheckedErrors(response, null, null); - return response.contains("messages=\"\""); + String url = query + "meta=userinfo&uiprop=hasmsg"; + return fetch(url, "hasNewMessages").contains("messages=\"\""); } /** - * Determines the current database replication lag. This method does not - * wait if the maxlag setting is exceeded. This method is thread safe. + * Determines the current database replication lag. * @return the current database replication lag * @throws IOException if a network error occurs * @see #setMaxLag @@ -1299,26 +1132,12 @@ public boolean hasNewMessages() throws IOException * MediaWiki documentation * @since 0.10 */ - public double getCurrentDatabaseLag() throws IOException + public int getCurrentDatabaseLag() throws IOException { - Map getparams = new HashMap<>(); - getparams.put("action", "query"); - getparams.put("meta", "siteinfo"); - getparams.put("siprop", "dbrepllag"); - - synchronized (this) - { - // bypass lag check for this request - int temp = getMaxLag(); - setMaxLag(-1); - String line = makeApiCall(getparams, null, "getCurrentDatabaseLag"); - detectUncheckedErrors(line, null, null); - setMaxLag(temp); - - String lag = parseAttribute(line, "lag", 0); - log(Level.INFO, "getCurrentDatabaseLag", "Current database replication lag is " + lag + " seconds"); - return Double.parseDouble(lag); - } + String line = fetch(query + "meta=siteinfo&siprop=dbrepllag", "getCurrentDatabaseLag"); + String lag = parseAttribute(line, "lag", 0); + log(Level.INFO, "getCurrentDatabaseLag", "Current database replication lag is " + lag + " seconds"); + return Integer.parseInt(lag); } /** @@ -1333,12 +1152,7 @@ public double getCurrentDatabaseLag() throws IOException */ public Map getSiteStatistics() throws IOException { - Map getparams = new HashMap<>(); - getparams.put("action", "query"); - getparams.put("meta", "siteinfo"); - getparams.put("siprop", "statistics"); - String text = makeApiCall(getparams, null, "getSiteStatistics"); - detectUncheckedErrors(text, null, null); + String text = fetch(query + "meta=siteinfo&siprop=statistics", "getSiteStatistics"); Map ret = new HashMap<>(20); ret.put("pages", Integer.parseInt(parseAttribute(text, "pages", 0))); ret.put("articles", Integer.parseInt(parseAttribute(text, "articles", 0))); @@ -1351,8 +1165,10 @@ public Map getSiteStatistics() throws IOException } /** - * Renders the specified wiki markup as HTML by passing it to the MediaWiki - * parser through the API. + * Renders the specified wiki markup by passing it to the MediaWiki + * parser through the API. (Note: this isn't implemented locally because + * I can't be stuffed porting Parser.php). One use of this method is to + * emulate the previewing functionality of the MediaWiki software. * * @param markup the markup to parse * @return the parsed markup as HTML @@ -1361,99 +1177,32 @@ public Map getSiteStatistics() throws IOException */ public String parse(String markup) throws IOException { - Map content = new HashMap<>(); - content.put("text", markup); - return parse(content, -1, false); + // This is POST because markup can be arbitrarily large, as in the size + // of an article (over 10kb). + String response = post(apiUrl + "action=parse", "prop=text&text=" + encode(markup, false), "parse"); + int y = response.indexOf('>', response.indexOf(""); + return decode(response.substring(y, z)); } /** - * Parses wikitext, revisions or pages. Deleted pages and revisions to - * deleted pages are not allowed if you don't have the rights to view them. - * - *

- * The returned HTML does not include "edit" links. Hyperlinks are - * rewritten from useless relative links to other wiki pages to full URLs. - * References to resources using protocol relative URLs are rewritten to - * use {@linkplain #getProtocol() this wiki's protocol}. - * - *

- * Warnings: - *

    - *
  • The parameters to this method will be changed when the time comes - * for JDK11 refactoring to accept Map.Entry instead. I also haven't - * decided how many more boolean parameters to add, and what format - * they will take. - *
+ * Same as {@link #parse(java.lang.String)}, but also strips out unwanted + * crap. This might be useful to subclasses. * - * @param content a Map following the same scheme as specified by {@link - * #diff(Map, Map)} - * @param section parse only this section (optional, use -1 to skip) - * @param nolimitreport do not include the HTML comment detailing limits - * @return the parsed wikitext - * @throws NoSuchElementException or IllegalArgumentException if no content - * was supplied for parsing - * @throws SecurityException if you pass a RevisionDeleted revision and - * lack the necessary privileges - * @throws IOException if a network error occurs - * @see #parse(String) - * @see #getRenderedText(String) - * @see Wiki.Revision#getRenderedText() - * @see MediaWiki - * documentation - * @since 0.35 - */ - public String parse(Map content, int section, boolean nolimitreport) throws IOException - { - Map getparams = new HashMap<>(); - getparams.put("action", "parse"); - getparams.put("prop", "text"); - if (nolimitreport) - getparams.put("disablelimitreport", "1"); - getparams.put("disableeditsection", "1"); - Map postparams = new HashMap<>(); - - Map.Entry entry = content.entrySet().iterator().next(); - Object value = entry.getValue(); - switch (entry.getKey()) - { - case "title": - getparams.put("page", normalize((String)value)); - break; - case "revid": - getparams.put("oldid", value.toString()); - break; - case "revision": - getparams.put("oldid", String.valueOf(((Revision)value).getID())); - break; - case "text": - postparams.put("text", value); - break; - default: - throw new IllegalArgumentException("No content was specified to parse!"); - } - if (section >= 0) - getparams.put("section", String.valueOf(section)); - - String response = makeApiCall(getparams, postparams, "parse"); - Consumer noop = desc -> {}; - Map> warnings = Map.of( - "missingtitle", noop, - "missingcontent", noop, - "nosuchsection", noop, - "nosuchrevid", noop); - if (detectUncheckedErrors(response, null, warnings)) - { - int y = response.indexOf('>', response.indexOf(""); - - // Rewrite URLs to replace useless relative links and make images work on - // locally saved copies of wiki pages. - String html = decode(response.substring(y, z)); - html = html.replace("href=\"/wiki", "href=\"" + protocol + domain + "/wiki"); - html = html.replace(" src=\"//", " src=\"" + protocol); // a little fragile for my liking, but will do - return html; - } - return null; + * @param in the string to parse + * @return that string without the crap + * @throws IOException if a network error occurs + * @since 0.14 + */ + protected String parseAndCleanup(String in) throws IOException + { + String output = parse(in); + output = output.replace("

", "").replace("

", ""); // remove paragraph tags + output = output.replace("\n", ""); // remove new lines + + // strip out the parser report, which comes at the end + int a = output.indexOf("" nocreate="" allowusertalk=""/> - for (int a = line.indexOf(" 0; a = line.indexOf("", a), line.indexOf("", a)); - String temp = line.substring(a, b); - - String blocker = parseAttribute(temp, "by", 0); - String blockeduser = parseAttribute(temp, "user", 0); - String target; - if (blockeduser == null) // autoblock - target = "#" + parseAttribute(temp, "id", 0); - else - target = namespaceIdentifier(USER_NAMESPACE) + ":" + blockeduser; + url.append("&bkstart="); + url.append(start.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } + if (!user.isEmpty()) + { + url.append("&bkusers="); + url.append(user); + } - LogEntry le = parseLogEntry(temp, blocker, BLOCK_LOG, "block", target); - results.add(le); - } - }; - List entries = new ArrayList<>(); - if (users == null) - entries.addAll(makeListQuery("bk", getparams, null, "getBlockList", limit, parser)); - else + // connection + List entries = queryAPIResult("bk", url, "getIPBlockList", (line, results) -> { - for (String bkusers : constructTitleString(users)) + try + { + // XML form: + for (int a = line.indexOf(" 0; a = line.indexOf("", a); + String temp = line.substring(a, b); + LogEntry le = parseLogEntry(temp); + le.type = BLOCK_LOG; + le.action = "block"; + // parseLogEntries parses block target into le.user due to mw.api + // attribute name + if (le.user == null) // autoblock + le.target = "#" + parseAttribute(temp, "id", 0); + else + le.target = namespaceIdentifier(USER_NAMESPACE) + ":" + le.user.username; + // parse blocker for real + le.user = new User(parseAttribute(temp, "by", 0)); + results.add(le); + } + } + catch (IOException ex) { - getparams.put("bkusers", bkusers); - entries.addAll(makeListQuery("bk", getparams, null, "getBlockList", limit, parser)); + throw new UncheckedIOException(ex); } + }); + + // log statement + StringBuilder logRecord = new StringBuilder("Successfully fetched IP block list "); + if (!user.isEmpty()) + { + logRecord.append(" for "); + logRecord.append(user); + } + if (start != null) + { + logRecord.append(" from "); + logRecord.append(start.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } + if (end != null) + { + logRecord.append(" to "); + logRecord.append(end.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); } - log(Level.INFO, "getBlockList", "Successfully fetched block list " + entries.size() + " entries)"); - return entries; + int size = entries.size(); + logRecord.append(" ("); + logRecord.append(size); + logRecord.append(" entries)"); + log(Level.INFO, "getIPBlockList", logRecord.toString()); + return entries.toArray(new LogEntry[size]); + } + + /** + * Gets the log entries representing actions that were performed on a + * specific target. Equivalent to [[Special:Log]]. + * @param logtype what log to get (e.g. {@link #DELETION_LOG}) + * @param action what action to get (e.g. delete, undelete, etc.), use + * null to not specify one + * @param user the user performing the action. Use null not to specify + * one. + * @param target the target of the action(s). + * @throws IOException if a network error occurs + * @return the specified log entries + * @since 0.08 + */ + public LogEntry[] getLogEntries(String logtype, String action, String user, String target) throws IOException + { + return getLogEntries(logtype, action, user, target, null, null, Integer.MAX_VALUE, ALL_NAMESPACES); + } + + /** + * Gets the last how ever many log entries in the specified log. Equivalent + * to [[Special:Log]] and [[Special:Newimages]] when + * type.equals({@link #UPLOAD_LOG}). + * + * @param logtype what log to get (e.g. {@link #DELETION_LOG}) + * @param action what action to get (e.g. delete, undelete, etc.), use + * null to not specify one + * @param amount the number of entries to get (overrides global limits) + * @throws IOException if a network error occurs + * @throws IllegalArgumentException if the log type doesn't exist + * @return the specified log entries + */ + public LogEntry[] getLogEntries(String logtype, String action, int amount) throws IOException + { + return getLogEntries(logtype, action, null, null, null, null, amount, ALL_NAMESPACES); } /** * Gets the specified amount of log entries between the given times by * the given user on the given target. Equivalent to [[Special:Log]]. - * Accepted parameters from helper are: - * - *
    - *
  • {@link Wiki.RequestHelper#withinDateRange(OffsetDateTime, - * OffsetDateTime) date range} - *
  • {@link Wiki.RequestHelper#byUser(String) user} - *
  • {@link Wiki.RequestHelper#byTitle(String) title} - *
  • {@link Wiki.RequestHelper#reverse(boolean) reverse} - *
  • {@link Wiki.RequestHelper#inNamespaces(int...) namespaces} (one - * namespace only, must not be used if a title is specified) - *
  • {@link Wiki.RequestHelper#taggedWith(String) tag} - *
  • {@link Wiki.RequestHelper#limitedTo(int) local query limit} - *
+ * WARNING: the start date is the most recent of the dates given, and + * the order of enumeration is from newest to oldest. * * @param logtype what log to get (e.g. {@link #DELETION_LOG}) - * @param action what action to get (e.g. delete, undelete, etc.), use - * {@code null} to not specify one - * @param helper a {@link Wiki.RequestHelper} (optional, use null to not - * provide any of the optional parameters noted above) + * @param action what action to get (e.g. delete, undelete, etc.), use + * null to not specify one + * @param user the user performing the action. Use null not to specify + * one. + * @param target the target of the action. Use null not to specify one. + * @param start what timestamp to start. Use null to not specify one. + * @param end what timestamp to end. Use null to not specify one. + * @param amount the amount of log entries to get. If both start and + * end are defined, this is ignored. Use Integer.MAX_VALUE to not + * specify one (overrides global limits) + * @param namespace filters by namespace. Returns empty if namespace + * doesn't exist. Use {@link #ALL_NAMESPACES} to not specify one. * @throws IOException if a network error occurs - * @throws SecurityException if the user lacks the credentials needed to - * access a privileged log + * @throws IllegalArgumentException if start < end or amount < 1 * @return the specified log entries * @since 0.08 */ - public List getLogEntries(String logtype, String action, Wiki.RequestHelper helper) throws IOException + public LogEntry[] getLogEntries(String logtype, String action, String user, String target, + OffsetDateTime start, OffsetDateTime end, int amount, int namespace) throws IOException { - int limit = -1; - Map getparams = new HashMap<>(); - getparams.put("list", "logevents"); - getparams.put("leprop", "ids|title|type|user|timestamp|comment|parsedcomment|details|tags"); + // construct the query url from the parameters given + StringBuilder url = new StringBuilder(query); + url.append("list=logevents&leprop=ids%7Ctitle%7Ctype%7Cuser%7Ctimestamp%7Ccomment%7Cdetails"); + + // check for amount + if (amount < 1) + throw new IllegalArgumentException("Tried to retrieve less than one log entry!"); + if (!logtype.equals(ALL_LOGS)) { if (action == null) - getparams.put("letype", logtype); + { + url.append("&letype="); + url.append(logtype); + } else - getparams.put("leaction", logtype + "/" + action); + { + url.append("&leaction="); + url.append(logtype); + url.append("/"); + url.append(action); + } + } + if (namespace != ALL_NAMESPACES) + { + url.append("&lenamespace="); + url.append(namespace); + } + if (user != null) + { + url.append("&leuser="); + url.append(encode(user, true)); + } + if (target != null) + { + url.append("&letitle="); + url.append(encode(target, true)); } - if (helper != null) + if (start != null) { - helper.setRequestType("le"); - getparams.putAll(helper.addTitleParameter()); - getparams.putAll(helper.addDateRangeParameters()); - getparams.putAll(helper.addUserParameter()); - getparams.putAll(helper.addReverseParameter()); - getparams.putAll(helper.addNamespaceParameter()); - getparams.putAll(helper.addTagParameter()); - limit = helper.limit(); + if (end != null && start.isBefore(end)) //aargh + throw new IllegalArgumentException("Specified start date is before specified end date!"); + url.append("&lestart="); + url.append(start.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + } + if (end != null) + { + url.append("&leend="); + url.append(end.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); } - List entries = makeListQuery("le", getparams, null, "getLogEntries", limit, (line, results) -> + int originallimit = getQueryLimit(); + setQueryLimit(Math.min(amount, originallimit)); + List entries = queryAPIResult("le", url, "getLogEntries", (line, results) -> { String[] items = line.split("getLogEntries(), - * getImageHistory() and getBlockList() into {@link Wiki.LogEntry} + * Parses xml generated by getLogEntries(), + * getImageHistory() and getIPBlockList() into LogEntry * objects. Override this if you want custom log types. NOTE: if * RevisionDelete was used on a log entry, the relevant values will be * null. * * @param xml the xml to parse - * @param user null, or use this value for the performer of the log entry - * @param type null, or use this value for the type of the log entry - * @param action null, or use this value for the action of the log entry - * @param target null, or user this value for the target of the log entry * @return the parsed log entry * @since 0.18 */ - protected LogEntry parseLogEntry(String xml, String user, String type, String action, String target) + protected LogEntry parseLogEntry(String xml) { - // ID (getLogEntries only) - long id = -1; - if (xml.contains("logid=\"")) - id = Long.parseLong(parseAttribute(xml, "logid", 0)); - + // note that we can override these in the calling method + String type = "", action = ""; boolean actionhidden = xml.contains("actionhidden=\""); - if (type == null && xml.contains("type=\"")) // only getLogEntries + if (xml.contains("type=\"")) // only getLogEntries { type = parseAttribute(xml, "type", 0); action = parseAttribute(xml, "action", 0); } // reason - String reason, parsedreason; + String reason; boolean reasonhidden = xml.contains("commenthidden=\""); - if (USER_CREATION_LOG.equals(type)) - { - // there is no reason for creating a user + if (type.equals(USER_CREATION_LOG)) // there is no reason for creating a user reason = ""; - parsedreason = ""; - } else if (xml.contains("reason=\"")) - { reason = parseAttribute(xml, "reason", 0); - parsedreason = null; // not available in list=blocks / getBlockList! - } else - { reason = parseAttribute(xml, "comment", 0); - parsedreason = parseAttribute(xml, "parsedcomment", 0); - } - // generic performer name + // generic performer name (won't work for ipblocklist, overridden there) boolean userhidden = xml.contains("userhidden=\"\""); - if (user == null && xml.contains("user=\"")) - user = parseAttribute(xml, "user", 0); + User performer = null; + if (xml.contains("user=\"")) + performer = new User(parseAttribute(xml, "user", 0)); // generic target name - // space is important -- commons.getImageHistory("File:Chief1.gif"); - if (target == null && xml.contains(" title=\"")) + String target = null; + if (xml.contains(" title=\"")) // space is important -- commons.getImageHistory("File:Chief1.gif"); target = parseAttribute(xml, "title", 0); OffsetDateTime timestamp = OffsetDateTime.parse(parseAttribute(xml, "timestamp", 0)); - // details - Map details = new HashMap<>(); + // details: TODO: make this a HashMap + Object details = null; if (xml.contains("commenthidden")) // oversighted details = null; else if (type.equals(MOVE_LOG)) - details.put("target_title", parseAttribute(xml, "target_title", 0)); + details = parseAttribute(xml, "target_title", 0); // the new title else if (type.equals(BLOCK_LOG) || xml.contains(" 10) // not an unblock { int d = s.indexOf('\"', c); - if (s.contains("anononly")) // anon-only - details.put("anononly", "true"); - if (s.contains("nocreate")) // account creation blocked - details.put("nocreate", "true"); - if (s.contains("noautoblock")) // autoblock disabled - details.put("noautoblock", "true"); - if (s.contains("noemail")) // email disabled - details.put("noemail", "true"); - if (s.contains("nousertalk")) // cannot edit talk page - details.put("nousertalk", "true"); - details.put("expiry", s.substring(c, d)); - - // partial block parameters - if (s.contains("")) + details = new Object[] { - details.put("partial", "true"); - if (s.contains("")) - { - details.put("type", "namespaces"); - // TODO: add the actual namespaces - } - else if (s.contains("")) - { - details.put("type", "pages"); - // TODO: add the actual pages - } - } + s.contains("anononly"), // anon-only + s.contains("nocreate"), // account creation blocked + s.contains("noautoblock"), // autoblock disabled + s.contains("noemail"), // email disabled + s.contains("nousertalk"), // cannot edit talk page + s.substring(c, d) // duration + }; } } else if (type.equals(PROTECTION_LOG)) { - if (action.equals("protect")) + if (action.equals("unprotect")) + details = null; + else { // FIXME: return a protectionstate here? - details.put("protection string", parseAttribute(xml, "description", 0)); + int a = xml.indexOf("") + 7; + int b = xml.indexOf("", a); + details = xml.substring(a, b); } } else if (type.equals(USER_RENAME_LOG)) { int a = xml.indexOf("") + 7; int b = xml.indexOf("", a); - details.put("new username", decode(xml.substring(a, b))); + details = decode(xml.substring(a, b)); // the new username } else if (type.equals(USER_RIGHTS_LOG)) { - int a = xml.indexOf(""); - if (a != -1) { - int b = xml.indexOf("", a); - var oldgroups = xml.substring(a + 11, b); - var old_list = new ArrayList(); - for (int end = 0, start = oldgroups.indexOf(""); start != -1; start = oldgroups.indexOf("", end)) { - end = oldgroups.indexOf("", start); - old_list.add(oldgroups.substring(start + 3, end)); - } - details.put("oldgroups", String.join(",", old_list)); - } else // self-closing empty "" tag - details.put("oldgroups", ""); - int c = xml.indexOf(""); - if (c != -1) { - int d = xml.indexOf("", c); - var newgroups = xml.substring(c + 11, d); - var new_list = new ArrayList(); - for (int end = 0, start = newgroups.indexOf(""); start != -1; start = newgroups.indexOf("", end)) { - end = newgroups.indexOf("", start); - new_list.add(newgroups.substring(start + 3, end)); - } - details.put("newgroups", String.join(",", new_list)); - } else // self-closing empty "" tag - details.put("newgroups", ""); - } - - // tags - List tags = new ArrayList<>(); - if (xml.contains("")) - { - for (int idx = xml.indexOf(""); idx >= 0; idx = xml.indexOf("", ++idx)) - tags.add(xml.substring(idx + 5, xml.indexOf("", idx))); + int a = xml.indexOf("new=\"") + 5; + int b = xml.indexOf('\"', a); + StringTokenizer tk = new StringTokenizer(xml.substring(a, b), ", "); + List temp = new ArrayList<>(); + while (tk.hasMoreTokens()) + temp.add(tk.nextToken()); + details = temp.toArray(new String[temp.size()]); } - - LogEntry le = new LogEntry(id, timestamp, user, reason, parsedreason, type, action, target, details); - le.setUserDeleted(userhidden); - le.setCommentDeleted(reasonhidden); - le.setContentDeleted(actionhidden); - le.setTags(tags); + + LogEntry le = new LogEntry(type, action, reason, performer, target, timestamp, details); + le.userDeleted = userhidden; + le.reasonDeleted = reasonhidden; + le.targetDeleted = actionhidden; return le; } @@ -6085,7 +5424,7 @@ else if (type.equals(USER_RIGHTS_LOG)) * @throws IOException if a network error occurs * @since 0.15 */ - public List prefixIndex(String prefix) throws IOException + public String[] prefixIndex(String prefix) throws IOException { return listPages(prefix, null, ALL_NAMESPACES, -1, -1, null); } @@ -6098,7 +5437,7 @@ public List prefixIndex(String prefix) throws IOException * @throws IOException if a network error occurs * @since 0.15 */ - public List shortPages(int cutoff) throws IOException + public String[] shortPages(int cutoff) throws IOException { return listPages("", null, MAIN_NAMESPACE, -1, cutoff, null); } @@ -6112,7 +5451,7 @@ public List shortPages(int cutoff) throws IOException * @return pages below that size in that namespace * @since 0.15 */ - public List shortPages(int cutoff, int namespace) throws IOException + public String[] shortPages(int cutoff, int namespace) throws IOException { return listPages("", null, namespace, -1, cutoff, null); } @@ -6125,7 +5464,7 @@ public List shortPages(int cutoff, int namespace) throws IOException * @throws IOException if a network error occurs * @since 0.15 */ - public List longPages(int cutoff) throws IOException + public String[] longPages(int cutoff) throws IOException { return listPages("", null, MAIN_NAMESPACE, cutoff, -1, null); } @@ -6139,7 +5478,7 @@ public List longPages(int cutoff) throws IOException * @throws IOException if a network error occurs * @since 0.15 */ - public List longPages(int cutoff, int namespace) throws IOException + public String[] longPages(int cutoff, int namespace) throws IOException { return listPages("", null, namespace, cutoff, -1, null); } @@ -6161,7 +5500,7 @@ public List longPages(int cutoff, int namespace) throws IOException * @since 0.09 * @throws IOException if a network error occurs */ - public List listPages(String prefix, Map protectionstate, int namespace) throws IOException + public String[] listPages(String prefix, Map protectionstate, int namespace) throws IOException { return listPages(prefix, protectionstate, namespace, -1, -1, null); } @@ -6179,7 +5518,7 @@ public List listPages(String prefix, Map protectionstate * @param protectionstate a {@link #protect protection state}, use null * to not specify one * @param namespace a namespace. ALL_NAMESPACES is not suppported, an - * UnsupportedOperationException will be thrown unless a prefix is specified. + * UnsupportedOperationException will be thrown. * @param minimum the minimum size in bytes these pages can be. Use -1 to * not specify one. * @param maximum the maximum size in bytes these pages can be. Use -1 to @@ -6190,96 +5529,116 @@ public List listPages(String prefix, Map protectionstate * @since 0.09 * @throws IOException if a network error occurs */ - public List listPages(String prefix, Map protectionstate, int namespace, int minimum, + public String[] listPages(String prefix, Map protectionstate, int namespace, int minimum, int maximum, Boolean redirects) throws IOException { + // @revised 0.15 to add short/long pages // No varargs namespace here because MW API only supports one namespace // for this module. - Map getparams = new HashMap<>(); - getparams.put("list", "allpages"); - if (!prefix.isEmpty()) + StringBuilder url = new StringBuilder(query); + url.append("list=allpages"); + if (!prefix.isEmpty()) // prefix { - if (namespace == ALL_NAMESPACES) - { - namespace = namespace(prefix); - prefix = removeNamespace(prefix); - } - getparams.put("apprefix", normalize(prefix)); + // cull the namespace prefix + namespace = namespace(prefix); + if (prefix.contains(":") && namespace != MAIN_NAMESPACE) + prefix = prefix.substring(prefix.indexOf(':') + 1); + url.append("&apprefix="); + url.append(encode(prefix, true)); } else if (namespace == ALL_NAMESPACES) // check for namespace throw new UnsupportedOperationException("ALL_NAMESPACES not supported in MediaWiki API."); - getparams.put("apnamespace", String.valueOf(namespace)); + url.append("&apnamespace="); + url.append(namespace); if (protectionstate != null) { - StringBuilder apprtype = new StringBuilder(); - StringBuilder apprlevel = new StringBuilder(); - protectionstate.forEach((key, value) -> + StringBuilder apprtype = new StringBuilder("&apprtype="); + StringBuilder apprlevel = new StringBuilder("&apprlevel="); + for (Map.Entry entry : protectionstate.entrySet()) { + String key = entry.getKey(); if (key.equals("cascade")) - getparams.put("apprfiltercascade", (Boolean)value ? "cascading" : "noncascading"); + { + url.append("&apprfiltercascade="); + url.append((Boolean)entry.getValue() ? "cascading" : "noncascading"); + } else if (!key.contains("expiry")) { apprtype.append(key); - apprtype.append("|"); - apprlevel.append(value); - apprlevel.append("|"); + apprtype.append("%7C"); + apprlevel.append((String)entry.getValue()); + apprlevel.append("%7C"); } - }); - getparams.put("apprtype", apprtype.substring(0, apprtype.length() - 1)); - getparams.put("apprlevel", apprlevel.substring(0, apprlevel.length() - 1)); + } + apprtype.delete(apprtype.length() - 3, apprtype.length()); + apprlevel.delete(apprlevel.length() - 3, apprlevel.length()); + url.append(apprtype); + url.append(apprlevel); } // max and min - if (minimum >= 0) - getparams.put("apminsize", String.valueOf(minimum)); - if (maximum >= 0) - getparams.put("apmaxsize", String.valueOf(maximum)); + if (minimum != -1) + { + url.append("&apminsize="); + url.append(minimum); + } + if (maximum != -1) + { + url.append("&apmaxsize="); + url.append(maximum); + } if (redirects != null) - getparams.put("apfilterredir", redirects ? "redirects" : "nonredirects"); + { + url.append("&apfilterredir="); + url.append(redirects ? "redirects" : "nonredirects"); + } // set query limit = 1 request if max, min, prefix or protection level // not specified - int limit = -1; + int originallimit = getQueryLimit(); if (maximum < 0 && minimum < 0 && prefix.isEmpty() && protectionstate == null) - limit = max; - List pages = makeListQuery("ap", getparams, null, "listPages", limit, (line, results) -> + setQueryLimit(max); + List pages = queryAPIResult("ap", url, "listPages", (line, results) -> { // xml form:

for (int a = line.indexOf("

0; a = line.indexOf("

Warnings: - *

- * - * @param page one of the qppage values specifed by the documentation below - * (case sensitive) + * WARNING: some of these may be *CACHED*, *DISABLED* and/or *LIMITED* on + * large wikis. + * + * @param page one of { Ancientpages, BrokenRedirects, Deadendpages, + * Disambiguations, DoubleRedirects, Listredirects, Lonelypages, Longpages, + * Mostcategories, Mostimages, Mostinterwikis, Mostlinkedcategories, + * Mostlinkedtemplates, Mostlinked, Mostrevisions, Fewestrevisions, Shortpages, + * Uncategorizedcategories, Uncategorizedpages, Uncategorizedimages, + * Uncategorizedtemplates, Unusedcategories, Unusedimages, Wantedcategories, + * Wantedfiles, Wantedpages, Wantedtemplates, Unwatchedpages, Unusedtemplates, + * Withoutinterwiki }. This parameter is *case sensitive*. * @return the list of pages returned by that particular special page * @throws IOException if a network error occurs - * @throws SecurityException if the user lacks the privileges necessary to - * view a report (e.g. unwatchedpages) + * @throws CredentialNotFoundException if page=Unwatchedpages and we cannot + * read it * @since 0.28 - * @see MediaWiki - * documentation */ - public List queryPage(String page) throws IOException + public String[] queryPage(String page) throws IOException, CredentialNotFoundException { - Map getparams = new HashMap<>(); - getparams.put("list", "querypage"); - getparams.put("qppage", page); + if (page.equals("Unwatchedpages") && (user == null || !user.isAllowedTo("unwatchedpages"))) + throw new CredentialNotFoundException("User does not have the \"unwatchedpages\" permission."); - List pages = makeListQuery("qp", getparams, null, "queryPage", -1, (line, results) -> + StringBuilder url = new StringBuilder(query); + url.append("action=query&list=querypage&qppage="); + url.append(page); + + List pages = queryAPIResult("qp", url, "queryPage", (line, results) -> { // xml form: for (int x = line.indexOf(" 0; x = line.indexOf(" queryPage(String page) throws IOException int temp = pages.size(); log(Level.INFO, "queryPage", "Successfully retrieved [[Special:" + page + "]] (" + temp + " pages)"); - return pages; + return pages.toArray(new String[temp]); } /** - * Fetches recently created pages. See {@link #recentChanges(Wiki.RequestHelper, - * String)} for full documentation. Equivalent to [[Special:Newpages]]. + * Fetches the amount most recently created pages in the main + * namespace. WARNING: The + * recentchanges table stores new pages for a finite period of time; + * it is not possible to retrieve pages created before then. * - * @param helper a {@link Wiki.RequestHelper} (optional, use null to not - * provide any optional parameters + * @param amount the number of pages to fetch (overrides global query + * limits) + * @return the revisions that created the pages satisfying the requirements + * above + * @throws IOException if a network error occurs + * @since 0.20 + */ + public Revision[] newPages(int amount) throws IOException + { + return recentChanges(amount, 0, true, MAIN_NAMESPACE); + } + + /** + * Fetches the amount most recently created pages in the main + * namespace subject to the specified constraints. WARNING: The recentchanges + * table stores new pages for a finite period of + * time; it is not possible to retrieve pages created before then. + * Equivalent to [[Special:Newpages]]. + * + * @param rcoptions a bitmask of {@link #HIDE_ANON} etc that dictate which + * pages we return (e.g. to exclude patrolled pages set rcoptions = HIDE_PATROLLED). + * @param amount the amount of new pages to get (overrides global query + * limits) + * @param ns a list of namespaces to filter by, empty = all namespaces. * @return the revisions that created the pages satisfying the requirements * above * @throws IOException if a network error occurs - * @since 0.35 + * @since 0.20 */ - public List newPages(Wiki.RequestHelper helper) throws IOException + public Revision[] newPages(int amount, int rcoptions, int... ns) throws IOException { - return recentChanges(helper, "new"); + return recentChanges(amount, rcoptions, true, ns); } /** - * Fetches recent edits to this wiki. See {@link - * #recentChanges(Wiki.RequestHelper, String)} for full documentation. + * Fetches the amount most recent changes in the main namespace. + * WARNING: The + * recentchanges table stores new pages for a finite period of + * time; it is not possible to retrieve pages created before then. * Equivalent to [[Special:Recentchanges]]. + *

+ * Note: Log entries in recent changes have a revid of 0! * - * @param helper a {@link Wiki.RequestHelper} (optional, use null to not - * provide any optional parameters + * @param amount the number of entries to return (overrides global query + * limits) * @return the recent changes that satisfy these criteria * @throws IOException if a network error occurs * @since 0.23 */ - public List recentChanges(Wiki.RequestHelper helper) throws IOException + public Revision[] recentChanges(int amount) throws IOException { - return recentChanges(helper, null); + return recentChanges(amount, 0, false, MAIN_NAMESPACE); } /** - * Fetches recent changes to this wiki. WARNING: The recentchanges - * table stores edits for a - * finite period of time; it is not possible to retrieve pages created - * before then. Equivalent to [[Special:Recentchanges]]. - * + * Fetches the amount most recent changes in the specified + * namespace. WARNING: The recent changes table only stores new pages for + * about a month. It is not possible to retrieve changes before then. + * Equivalent to [[Special:Recentchanges]]. *

- * Accepted parameters from helper are: - *

    - *
  • {@link Wiki.RequestHelper#withinDateRange(OffsetDateTime, - * OffsetDateTime) date range} - *
  • {@link Wiki.RequestHelper#byUser(String) user} - *
  • {@link Wiki.RequestHelper#notByUser(String) not by user} - *
  • {@link Wiki.RequestHelper#reverse(boolean) reverse} - *
  • {@link Wiki.RequestHelper#inNamespaces(int...) namespaces} - *
  • {@link Wiki.RequestHelper#taggedWith(String) tag} - *
  • {@link Wiki.RequestHelper#filterBy(Map) filter by}: "minor", "bot", - * "anon", "redirect", "patrolled" - *
  • {@link Wiki.RequestHelper#limitedTo(int) local query limit} - *
+ * Note: Log entries in recent changes have a revid of 0! * + * @param amount the number of entries to return (overrides global query + * limits) + * @param ns a list of namespaces to filter by, empty = all namespaces. + * @return the recent changes that satisfy these criteria + * @throws IOException if a network error occurs + * @since 0.23 + */ + public Revision[] recentChanges(int amount, int[] ns) throws IOException + { + return recentChanges(amount, 0, false, ns); + } + + /** + * Fetches the amount most recent changes in the specified + * namespace subject to the specified constraints. WARNING: The recent + * changes table only stores new pages for about a month. It is not + * possible to retrieve changes before then. Equivalent to + * [[Special:Recentchanges]]. *

- * If {@code rctype} is not {@code "edit"} or {@code "new"} then the results - * consist of pseudo-revisions whose data does not correspond to an actual - * on-wiki state. For example: - * - *

    - *
  • {@code rctype == "log"} yields {@code id == 0} and {@code title} is - * the log entry target - *
  • {@code rctype == "external"} yields {@code id} as the most recent - * edit to {@code title}, {@code previous_id == id}, {@code user} is - * the external user making the change, {@code sizediff == 0} and - * {@code comment} describes the external change - *
  • {@code rctype =="categorize"} yields {@code title} as the category - * added or removed and {@code comment} specifies the page added or - * removed to that category - *
+ * Note: Log entries in recent changes have a revid of 0! * - * @param helper a {@link Wiki.RequestHelper} (optional, use null to not - * provide any of the optional parameters described above - * @param rctype null, "edit" (edits only) or "new" (new pages); your - * mileage may vary for other types (log, external, categorize) + * @param amount the number of entries to return (overrides global query + * limits) + * @param ns a list of namespaces to filter by, empty = all namespaces. + * @param rcoptions a bitmask of HIDE_ANON etc that dictate which pages + * we return. * @return the recent changes that satisfy these criteria * @throws IOException if a network error occurs - * @since 0.35 + * @since 0.23 */ - protected List recentChanges(Wiki.RequestHelper helper, String rctype) throws IOException + public Revision[] recentChanges(int amount, int rcoptions, int... ns) throws IOException { - int limit = -1; - Map getparams = new HashMap<>(); - getparams.put("list", "recentchanges"); - getparams.put("rcprop", "title|ids|user|timestamp|flags|comment|parsedcomment|sizes|sha1|tags"); - if (helper != null) - { - helper.setRequestType("rc"); - getparams.putAll(helper.addNamespaceParameter()); - getparams.putAll(helper.addUserParameter()); - getparams.putAll(helper.addExcludeUserParameter()); - getparams.putAll(helper.addDateRangeParameters()); - getparams.putAll(helper.addTagParameter()); - getparams.putAll(helper.addReverseParameter()); - getparams.putAll(helper.addShowParameter()); - limit = helper.limit(); - } - - if (rctype != null) - getparams.put("rctype", rctype); + return recentChanges(amount, rcoptions, false, ns); + } - List revisions = makeListQuery("rc", getparams, null, "recentChanges", limit, + /** + * Fetches the amount most recent changes in the specified + * namespace subject to the specified constraints. WARNING: The recent + * changes table only stores new pages for about a month. It is not + * possible to retrieve changes before then. Equivalent to + * [[Special:Recentchanges]]. + *

+ * Note: Log entries in recent changes have a revid of 0! + * + * @param amount the number of entries to return (overrides global + * query limits) + * @param ns a list of namespaces to filter by, empty = all namespaces. + * @param rcoptions a bitmask of HIDE_ANON etc that dictate which pages + * we return. + * @param newpages show new pages only + * @return the recent changes that satisfy these criteria + * @throws IOException if a network error occurs + * @since 0.23 + */ + protected Revision[] recentChanges(int amount, int rcoptions, boolean newpages, int... ns) throws IOException + { + StringBuilder url = new StringBuilder(query); + url.append("list=recentchanges&rcprop=title%7Cids%7Cuser%7Ctimestamp%7Cflags%7Ccomment%7Csizes%7Csha1"); + constructNamespaceString(url, "rc", ns); + if (newpages) + url.append("&rctype=new"); + // rc options + if (rcoptions > 0) + { + url.append("&rcshow="); + if ((rcoptions & HIDE_ANON) == HIDE_ANON) + url.append("!anon%7C"); + if ((rcoptions & HIDE_SELF) == HIDE_SELF) + url.append("!self%7C"); + if ((rcoptions & HIDE_MINOR) == HIDE_MINOR) + url.append("!minor%7C"); + if ((rcoptions & HIDE_PATROLLED) == HIDE_PATROLLED) + url.append("!patrolled%7C"); + if ((rcoptions & HIDE_BOT) == HIDE_BOT) + url.append("!bot%7C"); + // chop off last | + url.delete(url.length() - 3, url.length()); + } + + int originallimit = getQueryLimit(); + setQueryLimit(amount); + List revisions = queryAPIResult("rc", url, newpages ? "newPages" : "recentChanges", (line, results) -> { // xml form for (int i = line.indexOf(" 0; i = line.indexOf("", i); + int j = line.indexOf("/>", i); results.add(parseRevision(line.substring(i, j), "")); } }); + setQueryLimit(originallimit); - log(Level.INFO, "recentChanges", "Successfully retrieved recent changes (" + revisions.size() + " revisions)"); - return revisions; + int temp = revisions.size(); + log(Level.INFO, "recentChanges", "Successfully retrieved recent changes (" + temp + " revisions)"); + return revisions.toArray(new Revision[temp]); } /** * Fetches all pages that use interwiki links to the specified wiki and the - * page on that wiki that is linked to. For example, {@code - * getInterWikiBacklinks("testwiki")} may return: + * page on that wiki that is linked to. For example, + * getInterWikiBacklinks("testwiki") may return: * * { * { "Spam", "testwiki:Blah" }, @@ -6431,7 +5844,7 @@ protected List recentChanges(Wiki.RequestHelper helper, String rctype) * @throws IOException if a network error occurs * @since 0.23 */ - public List getInterWikiBacklinks(String prefix) throws IOException + public String[][] getInterWikiBacklinks(String prefix) throws IOException { return getInterWikiBacklinks(prefix, "|"); } @@ -6445,8 +5858,8 @@ public List getInterWikiBacklinks(String prefix) throws IOException * *

* Example: If [[Test]] and [[Spam]] both contain the interwiki link - * [[testwiki:Blah]] then {@code getInterWikiBacklinks("testwiki", "Blah");} - * will return (sorted by title) + * [[testwiki:Blah]] then getInterWikiBacklinks("testwiki", "Blah"); + * will return (sorted by title) * * { * { "Spam", "testwiki:Blah" }, @@ -6469,20 +5882,23 @@ public List getInterWikiBacklinks(String prefix) throws IOException * prefix (the MediaWiki API doesn't like this) * @since 0.23 */ - public List getInterWikiBacklinks(String prefix, String title) throws IOException + public String[][] getInterWikiBacklinks(String prefix, String title) throws IOException { // must specify a prefix if (title.equals("|") && prefix.isEmpty()) throw new IllegalArgumentException("Interwiki backlinks: title specified without prefix!"); - Map getparams = new HashMap<>(); - getparams.put("list", "iwbacklinks"); - getparams.put("iwblprefix", prefix); + StringBuilder url = new StringBuilder(query); + url.append("list=iwbacklinks&iwblprefix="); + url.append(prefix); if (!title.equals("|")) - getparams.put("iwbltitle", normalize(title)); - getparams.put("iwblprop", "iwtitle|iwprefix"); + { + url.append("&iwbltitle="); + url.append(title); + } + url.append("&iwblprop=iwtitle%7Ciwprefix"); - List links = makeListQuery("iwbl", getparams, null, "getInterWikiBacklinks", -1, (line, results) -> + List links = queryAPIResult("iwbl", url, "getInterWikiBacklinks", (line, results) -> { // xml form: for (int x = line.indexOf(" 0; x = line.indexOf(" getInterWikiBacklinks(String prefix, String title) throws }); } }); - + log(Level.INFO, "getInterWikiBacklinks", "Successfully retrieved interwiki backlinks (" + links.size() + " interwikis)"); - return links; + return links.toArray(new String[0][0]); } // INNER CLASSES - + /** * Subclass for wiki users. * @since 0.05 */ - public class User implements Comparable + public class User implements Cloneable, Serializable { - private final String username; - private final OffsetDateTime registration; - // user privileges (volatile, changes rarely) - private List rights; - private List groups; - private boolean blocked; - // user preferences (volatile, changes rarely) - private Gender gender; - private boolean emailable; - // volatile, changes often - private int editcount; + private String username; + private String[] rights = null; // cache + private String[] groups = null; // cache /** * Creates a new user object. Does not create a new user on the @@ -6525,27 +5933,11 @@ public class User implements Comparable * be called for anons. * * @param username the username of the user - * @param registration when the user was registered - * @param rights the rights this user has - * @param groups the groups this user belongs to - * @param gender the self-declared {@link Wiki.Gender Gender} of this user. - * @param emailable whether the user can be emailed through [[Special:Emailuser]] - * @param blocked whether this user is blocked - * @param editcount the internal edit count of this user * @since 0.05 */ - protected User(String username, OffsetDateTime registration, List rights, List groups, - Gender gender, boolean emailable, boolean blocked,int editcount) + protected User(String username) { - this.username = Objects.requireNonNull(username); - // can be null per https://phabricator.wikimedia.org/T24097 - this.registration = registration; - this.rights = Objects.requireNonNull(rights); - this.groups = Objects.requireNonNull(groups); - this.gender = gender; - this.emailable = emailable; - this.blocked = blocked; - this.editcount = editcount; + this.username = username; } /** @@ -6553,131 +5945,111 @@ protected User(String username, OffsetDateTime registration, List rights * @return this user's username * @since 0.08 */ - public final String getUsername() + public String getUsername() { return username; } /** - * Gets the date/time at which this user account was created. May be - * {@code null} per - * https://phabricator.wikimedia.org/T24097. + * Gets various properties of this user. Returns: + *

    + *
  • editcount: (int) {@link #countEdits()} the user's edit + * count + *
  • groups: (String[]) the groups the user is in (see + * [[Special:Listgrouprights]]) + *
  • rights: (String[]) the stuff the user can do + *
  • emailable: (Boolean) whether the user can be emailed + * through [[Special:Emailuser]] or emailUser() + *
  • blocked: (Boolean) whether the user is blocked + *
  • gender: (Wiki.Gender) the user's gender + *
  • created: (Calendar) when the user account was created + *
+ * * @return (see above) - * @since 0.35 + * @throws IOException if a network error occurs + * @since 0.24 */ - public final OffsetDateTime getRegistrationDate() + public Map getUserInfo() throws IOException { - return registration; + return Wiki.this.getUserInfo(new String[] { username })[0]; } /** - * Returns {@code true} if the user is allowed to perform the specified - * action(s). Read [[Special:Listgrouprights]] before using this! + * Returns true if the user is allowed to perform the specified action. + * Uses the rights cache. Read [[Special:Listgrouprights]] before using + * this! * @param right a specific action - * @param morerights additional actions to check - * @return whether the user is allowed to execute them + * @return whether the user is allowed to execute it * @since 0.24 + * @throws IOException if a network error occurs */ - public boolean isAllowedTo(String right, String... morerights) - { - List temp = new ArrayList<>(); - temp.add(right); - temp.addAll(List.of(morerights)); - return rights.containsAll(temp); + public boolean isAllowedTo(String right) throws IOException + { + // We can safely assume the user is allowed to { read, edit, create, + // writeapi }. + if (rights == null) + rights = (String[])getUserInfo().get("rights"); + for (String r : rights) + if (r.equals(right)) + return true; + return false; } /** - * Returns {@code true} if the user is a member of the specified group. + * Returns true if the user is a member of the specified group. Uses + * the groups cache. * @param group a specific group * @return whether the user is in it * @since 0.24 + * @throws IOException if a network error occurs */ - public boolean isA(String group) - { - return groups.contains(group); - } - - /** - * Returns the groups the user is a member of. See [[Special:Listgrouprights]]. - * Changes in this list do not propagate to this object or the wiki. - * @return (see above) - * @since 0.35 - */ - public List getGroups() - { - return new ArrayList<>(groups); - } - - /** - * Returns the specific permissions this user has. See [[Special:Listgrouprights]]. - * Changes in this list do not propagate to the object or the wiki. - * @return (see above) - * @since 0.35 - */ - public List getRights() - { - return new ArrayList<>(rights); - } - - /** - * Returns whether this user can be emailed through [[Special:Emailuser]]. - * @return (see above) - * @see #emailUser(Wiki.User, String, String, boolean) - * @since 0.35 - */ - public boolean canBeEmailed() + public boolean isA(String group) throws IOException { - return emailable; + if (groups == null) + groups = (String[])getUserInfo().get("groups"); + for (String g : groups) + if (g.equals(group)) + return true; + return false; } /** - * Returns the self-disclosed {@linkplain Wiki.Gender gender} of this - * user. - * @return (see above) - * @see Wiki.Gender - * @since 0.35 + * Returns a log of the times when the user has been blocked. + * @return records of the occasions when this user has been blocked + * @throws IOException if something goes wrong + * @since 0.08 */ - public Gender getGender() + public LogEntry[] blockLog() throws IOException { - return gender; + return Wiki.this.getLogEntries(Wiki.BLOCK_LOG, null, null, "User:" + username); } /** - * Determines whether this user is blocked at the time of construction. - * If you want a live check, look up the user on the {@linkplain - * #getBlockList list of blocks}. + * Determines whether this user is blocked by looking it up on the IP + * block list. * @return whether this user is blocked + * @throws IOException if we cannot retrieve the IP block list * @since 0.12 */ - public boolean isBlocked() + public boolean isBlocked() throws IOException { - return blocked; + // @revised 0.18 now check for errors after each edit, including blocks + return getIPBlockList(username, null, null).length != 0; } /** - * Fetches the internal edit count for this user at the time of - * construction, which includes all live edits and deleted edits after - * (I think) January 2007. If you want to count live edits only, - * compute the size of {@link User#contribs(int...) User.contribs()}. + * Fetches the internal edit count for this user, which includes all + * live edits and deleted edits after (I think) January 2007. If you + * want to count live edits only, use the slower + * int count = {@link User#contribs(int...) user.contribs()}.length;. * * @return the user's edit count + * @throws IOException if a network error occurs * @since 0.16 */ - public int countEdits() - { - return editcount; - } - - /** - * Returns a log of the times when the user has been blocked. - * @return records of the occasions when this user has been blocked - * @throws IOException if something goes wrong - * @since 0.08 - */ - public List blockLog() throws IOException + public int countEdits() throws IOException { - Wiki.RequestHelper rh = new RequestHelper().byTitle("User:" + username); - return Wiki.this.getLogEntries(Wiki.BLOCK_LOG, null, rh); + return (Integer)getUserInfo().get("editcount"); } /** @@ -6687,430 +6059,243 @@ public List blockLog() throws IOException * @throws IOException if a network error occurs * @since 0.17 */ - public List contribs(int... ns) throws IOException + public Revision[] contribs(int... ns) throws IOException { - Wiki.RequestHelper rh = new RequestHelper().inNamespaces(ns); - return Wiki.this.contribs(username, rh); + return Wiki.this.contribs(username, ns); } - + /** * Returns the list of logged actions performed by this user. * @param logtype what log to get ({@link Wiki#DELETION_LOG} etc.) - * @param action what action to get (e.g. delete, undelete), use + * @param action what action to get (e.g. delete, undelete), use * "" to not specify one * @return (see above) * @throws IOException if a network error occurs * @since 0.33 */ - public List getLogEntries(String logtype, String action) throws IOException + public LogEntry[] getLogEntries(String logtype, String action) throws IOException { - Wiki.RequestHelper rh = new RequestHelper().byUser(username); - return Wiki.this.getLogEntries(logtype, action, rh); + return Wiki.this.getLogEntries(logtype, action, username, null); } /** - * Tests whether this user is equal to another one. - * @param x another object - * @return whether the usernames of the users are equal + * Copies this user object. + * @return the copy + * @throws CloneNotSupportedException if the clone fails + * @since 0.08 */ @Override - public boolean equals(Object x) + public User clone() throws CloneNotSupportedException { - if (!(x instanceof User)) - return false; - User other = (User)x; - return Objects.equals(username, other.username) - && Objects.equals(registration, other.registration); + try + { + return (User)super.clone(); + } + catch (CloneNotSupportedException e) + { + return null; + } } /** - * Returns a hashcode of this user based on the username and - * registration date. - * @return see above + * Tests whether this user is equal to another one. + * @param x another object + * @return whether the usernames of the users are equal + * @since 0.08 */ @Override - public int hashCode() + public boolean equals(Object x) { - return username.hashCode() * 127 + registration.hashCode(); + return x instanceof User && username.equals(((User)x).username); } /** - * Enables sorting of users by their username. - * @param other some other user - * @return less than zero if this user is alphabetically before the - * other, 0 if they are the same and 1 if alphabetically after + * Returns a string representation of this user. + * @return see above + * @since 0.17 */ @Override - public int compareTo(User other) + public String toString() { - return username.compareTo(other.username); + StringBuilder temp = new StringBuilder("User[username="); + temp.append(username); + temp.append("groups="); + temp.append(groups != null ? Arrays.toString(groups) : "unset"); + temp.append("]"); + return temp.toString(); } /** - * Returns a string representation of this user. + * Returns a hashcode of this user. * @return see above + * @since 0.19 */ @Override - public String toString() + public int hashCode() { - return getClass().getName() - + "[username=" + username - + ",registration=" + (registration != null ? registration.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) : "unset") - + ",groups=" + Arrays.toString(groups.toArray()) + "]"; + return username.hashCode() * 2 + 1; } } /** - * A data super class for an event happening on a wiki such as a {@link - * Wiki.Revision} or a {@link Wiki.LogEntry}. - * @since 0.35 + * A wrapper class for an entry in a wiki log, which represents an action + * performed on the wiki. + * + * @see #getLogEntries + * @since 0.08 */ - public abstract class Event implements Comparable + public class LogEntry implements Comparable { - private final long id; + // internal data storage + private long logid = -1; + private String type; + private String action; + private final String reason; + private User user; + private String target; private final OffsetDateTime timestamp; - private final String user; - private final String title; - private final String comment; - private final String parsedcomment; - private List tags; - private boolean commentDeleted = false, userDeleted = false, - contentDeleted = false; + private Object details; + private boolean reasonDeleted = false, userDeleted = false, + targetDeleted = false; /** - * Creates a new Event record. - * @param id the unique ID of the event - * @param timestamp the timestamp at which it occurred - * @param user the user or IP address performing the event - * @param title the title of the page affected - * @param comment the comment left by the user when performing the - * event (e.g. an edit summary) - * @param parsedcomment comment, but parsed into HTML + * Creates a new log entry. WARNING: does not perform the action + * implied. Use Wiki.class methods to achieve this. + * + * @param type the type of log entry, one of {@link #USER_CREATION_LOG}, + * {@link #DELETION_LOG}, {@link #BLOCK_LOG}, etc. + * @param action the type of action that was performed e.g. "delete", + * "unblock", "overwrite", etc. + * @param reason why the action was performed + * @param user the user who performed the action + * @param target the target of the action + * @param timestamp the local time when the action was performed. + * We will convert this back into an OffsetDateTime + * @param details the details of the action (e.g. the new title of + * the page after a move was performed). + * @since 0.08 */ - protected Event(long id, OffsetDateTime timestamp, String user, String title, String comment, String parsedcomment) + protected LogEntry(String type, String action, String reason, User user, + String target, OffsetDateTime timestamp, Object details) { - this.id = id; - this.timestamp = Objects.requireNonNull(timestamp); + this.type = type; + this.action = action; + this.reason = reason; this.user = user; - this.title = title; - this.comment = comment; - // Rewrite parsedcomments to fix useless relative hyperlinks to - // other wiki pages - if (parsedcomment == null) - this.parsedcomment = null; - else - this.parsedcomment = parsedcomment.replace("href=\"/wiki", "href=\"" + protocol + domain + "/wiki"); - } - - /** - * Gets the unique ID of this event. For a {@link Wiki.Revision}, this - * number is referred to as the "oldid" or "revid" and should not be - * confused with "rcid" (which is the ID in the recentchanges table). - * For a {@link Wiki.LogEntry}, this value only makes sense if the - * record was obtained through {@link Wiki#getLogEntries(String, String, - * RequestHelper)} and overloads (other methods return pseudo-LogEntries). - * @return the ID of this revision - */ - public long getID() - { - return id; - } - - /** - * Gets the timestamp of this event. - * @return the timestamp of this event - */ - public OffsetDateTime getTimestamp() - { - return timestamp; - } - - /** - * Returns the user or anon who performed this event. You should pass - * this (if not an IP) to {@link #getUser} to obtain a {@link - * Wiki.User} object. Returns {@code null} if the user was - * RevisionDeleted and you lack the necessary privileges. - * @return the user or anon - */ - public String getUser() - { - return user; - } - - /** - * Sets a boolean flag that the user triggering this event has been - * RevisionDeleted in on-wiki records. - * @param deleted (see above) - * @see #isUserDeleted() - * @see #getUser() - */ - protected void setUserDeleted(boolean deleted) - { - userDeleted = deleted; - } - - /** - * Returns {@code true} if the user triggering this event is - * RevisionDeleted. - * @return (see above) - * @see #getUser() - */ - public boolean isUserDeleted() - { - return userDeleted; + this.target = target; + this.timestamp = timestamp; + this.details = details; } - + /** - * Returns the page affected by this event. May be {@code null} for - * certain types of LogEntry and/or if the LogEntry is RevisionDeleted - * and you don't have the ability to access it. + * Gets the ID of this log entry. Only available if retrieved by + * {@link Wiki#getLogEntries}, otherwise returns -1. * @return (see above) - * @see #isContentDeleted() - */ - public String getTitle() - { - return title; - } - - /** - * Gets the comment for this event in wikitext. If this is a {@link - * Wiki.Revision}, this is the edit summary. If this is a {@link - * Wiki.LogEntry}, this is the reason for the logged action. WARNING: - * returns {@code null} if the reason was RevisionDeleted and you lack - * the necessary privileges. - * @return the comment associated with the event - * @see #getParsedComment() + * @since 0.33 */ - public String getComment() + public long getLogID() { - return comment; + return logid; } /** - * Gets the comment for this event, with limited parsing into HTML. - * Hyperlinks in the returned HTML are rewritten from useless relative - * URLs to full URLs that point to the wiki page in question. Returns - * {@code null} if {@linkplain #isCommentDeleted() the comment was - * RevisionDeleted} and you lack the necessary privileges. - * - *

Warnings: - *

    - *
  • Not available through {@link #getBlockList}. - *
- * - * @return the comment associated with the event, parsed into HTML - * @see #getComment() + * Gets the type of log that this entry is in. + * @return one of {@link Wiki#DELETION_LOG}, {@link Wiki#BLOCK_LOG}, etc. + * @since 0.08 */ - public String getParsedComment() + public String getType() { - return parsedcomment; + return type; } /** - * Sets a boolean flag that the comment associated with this event has - * been RevisionDeleted in on-wiki records. - * @param deleted (see above) - * @see #getComment - * @see #getParsedComment() - * @see #isCommentDeleted() + * Gets a string description of the action performed, for example + * "delete", "protect", "overwrite", ... WARNING: returns null if the + * action was RevisionDeleted. + * @return the type of action performed + * @since 0.08 */ - protected void setCommentDeleted(boolean deleted) + public String getAction() { - commentDeleted = deleted; + return action; } - + /** - * Returns {@code true} if the comment is RevisionDeleted. + * Returns true if the target has been RevisionDeleted (action is hidden + * in the GUI but retrievable by the API). * @return (see above) - * @see #getComment - * @see #getParsedComment() + * @since 0.32 */ - public boolean isCommentDeleted() + public boolean isTargetDeleted() { - return commentDeleted; + return targetDeleted; } /** - * Sets a boolean flag that the content of this event has been - * RevisionDeleted. - * @param deleted (see above) - * @see #isContentDeleted() - */ - protected void setContentDeleted(boolean deleted) - { - contentDeleted = deleted; - } - - /** - * Returns {@code true} if the content of this event has been - * RevisionDeleted. For a {@link Wiki.LogEntry}, this refers to the - * page the logged action affects and the logged action performed (e.g. - * "unblock" or "delete"). - * @return (see above) - */ - public boolean isContentDeleted() - { - return contentDeleted; - } - - /** - * Returns the list of tags attached to this event. Modifying the - * return value does not affect this Revision object or the wiki state. - * @return (see above) - * @see MediaWiki - * documentation - * @since 0.37 + * Gets the reason supplied by the perfoming user when the action + * was performed. WARNING: returns null if the reason was + * RevisionDeleted and one does not have access to the content. + * @return the reason the action was performed + * @since 0.08 */ - public List getTags() + public String getReason() { - return new ArrayList<>(tags); + return reason; } /** - * Sets the list of tags attached to this event. Modifying the supplied - * list does not affect this Revision object or change on-wiki state. - * @param tags a list of change tags - * @see MediaWiki - * documentation - * @since 0.37 - */ - protected void setTags(List tags) - { - this.tags = new ArrayList<>(tags); - } - - /** - * Returns a String representation of this Event. Subclasses only need - * to lop off the trailing "]" and add their own fields when overriding - * this method. + * Returns true if the reason is RevisionDeleted. * @return (see above) + * @since 0.32 */ - @Override - public String toString() + public boolean isReasonDeleted() { - return getClass().getName() - + "[id=" + id - + ",timestamp=" + timestamp.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) - + ",user=\"" + Objects.toString(user, "[DELETED]") + "\"" - + ",userDeleted=" + userDeleted - + ",title=\"" + Objects.toString(title, "[null or deleted]") + "\"" - + ",comment=\"" + Objects.toString(comment, "[DELETED]") + "\"" - + ",commentDeleted=" + commentDeleted - + ",contentDeleted=" + contentDeleted + ']'; + return reasonDeleted; } /** - * Determines whether this Event is equal to some other object. This - * method checks the ID, timestamp, user, title and comment. - * @param other the other object to compare to - * @return whether this instance is equal to that object + * Gets the user object representing who performed the action. + * WARNING: returns null if the user was RevisionDeleted and one does + * not have access to the content. + * @return the user who performed the action. + * @since 0.08 */ - @Override - public boolean equals(Object other) + public User getUser() { - if (!(other instanceof Event)) - return false; - Event event = (Event)other; - return id == event.id - && Objects.equals(timestamp, event.timestamp) - && Objects.equals(user, event.user) - && Objects.equals(title, event.title) - && Objects.equals(comment, event.comment); + return user; } - + /** - * Returns a hash code for this object based on the ID, timestamp, - * user, title and comment. + * Returns true if the user who performed this LogEntry is + * RevisionDeleted. * @return (see above) + * @since 0.32 */ - @Override - public int hashCode() - { - int hc = Long.hashCode(id); - hc = 127 * hc + timestamp.hashCode(); - hc = 127 * hc + Objects.hashCode(user); - hc = 127 * hc + Objects.hashCode(title); - hc = 127 * hc + Objects.hashCode(comment); - return hc; - } - - /** - * Compares this event to another one based on the recentness of their - * timestamps (more recent = positive return value), then - * alphabetically by user. - * @param other the event to compare to - * @return the comparator value, negative if less, positive if greater - */ - @Override - public int compareTo(Wiki.Event other) - { - int result = timestamp.compareTo(other.timestamp); - if (result == 0 && user != null) - result = user.compareTo(other.user); - return result; - } - } - - /** - * A wrapper class for an entry in a wiki log, which represents an action - * performed on the wiki. - * - * @see #getLogEntries - * @since 0.08 - */ - public class LogEntry extends Event - { - private final String type; - private String action; - private Map details; - - /** - * Creates a new log entry. WARNING: does not perform the action - * implied. Use Wiki.class methods to achieve this. - * - * @param id the unique of this log entry - * @param timestamp the local time when the action was performed. - * @param user the user who performed the action - * @param comment why the action was performed - * @param parsedcomment like comment, but parsed into HTML - * @param type the type of log entry, one of {@link #USER_CREATION_LOG}, - * {@link #DELETION_LOG}, {@link #BLOCK_LOG}, etc. - * @param action the type of action that was performed e.g. "delete", - * "unblock", "overwrite", etc. - * @param target the target of the action - * @param details the details of the action (e.g. the new title of - * the page after a move was performed). - * @since 0.08 - */ - protected LogEntry(long id, OffsetDateTime timestamp, String user, String comment, - String parsedcomment, String type, String action, String target, Map details) + public boolean isUserDeleted() { - super(id, timestamp, user, target, comment, parsedcomment); - this.type = Objects.requireNonNull(type); - this.action = action; - this.details = details; + return userDeleted; } /** - * Gets the type of log that this entry is in. - * @return one of {@link Wiki#DELETION_LOG}, {@link Wiki#BLOCK_LOG}, etc. + * Gets the target of the action represented by this log entry. WARNING: + * returns null if the content was RevisionDeleted and one does not + * have access to the content. + * @return the target of this log entry * @since 0.08 */ - public String getType() + public String getTarget() { - return type; + return target; } /** - * Gets a string description of the action performed, for example - * "delete", "protect", "overwrite", ... WARNING: returns null if the - * action was RevisionDeleted. - * @return the type of action performed + * Gets the timestamp of this log entry. + * @return the timestamp of this log entry * @since 0.08 */ - public String getAction() + public OffsetDateTime getTimestamp() { - return action; + return timestamp; } /** @@ -7124,14 +6309,13 @@ public String getAction() * USER_RENAME_LOG * The new username * BLOCK_LOG - * new Object[] { boolean anononly, boolean nocreate, boolean + * new Object[] { boolean anononly, boolean nocreate, boolean * noautoblock, boolean noemail, boolean nousertalk, String duration } * USER_RIGHTS_LOG - * The old ("oldgroups") and new ("newgroups") user rights, - * comma-separated + * The new user rights (String[]) * PROTECTION_LOG * if action == "protect" or "modify" return the protection level - * (int, -2 if unrecognized) if action == "move_prot" return + * (int, -2 if unrecognized) if action == "move_prot" return * the old title, else null * Others or RevisionDeleted * null @@ -7145,7 +6329,7 @@ public String getAction() * @return the details of the log entry * @since 0.08 */ - public Map getDetails() + public Object getDetails() { return details; } @@ -7158,21 +6342,44 @@ public Map getDetails() @Override public String toString() { - StringBuilder s = new StringBuilder(super.toString()); - s.deleteCharAt(s.length() - 1); + StringBuilder s = new StringBuilder("LogEntry[logid="); + s.append(logid); s.append(",type="); s.append(type); s.append(",action="); - s.append(Objects.toString(action, "[DELETED]")); - s.append(",details="); - s.append(details); + s.append(action == null ? "[hidden]" : action); + s.append(",user="); + s.append(user == null ? "[hidden]" : user.getUsername()); + s.append(",timestamp="); + s.append(timestamp.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + s.append(",target="); + s.append(target == null ? "[hidden]" : target); + s.append(",reason=\""); + s.append(reason == null ? "[hidden]" : reason); + s.append("\",details="); + if (details instanceof Object[]) + s.append(Arrays.asList((Object[])details)); // crude formatting hack + else + s.append(details); s.append("]"); return s.toString(); } /** - * Determines whether two LogEntries are equal based on the underlying - * {@linkplain Event#equals(Object) Event}, type and action. + * Compares this log entry to another one based on the recentness + * of their timestamps. + * @param other the log entry to compare + * @return whether this object is equal to + * @since 0.18 + */ + @Override + public int compareTo(Wiki.LogEntry other) + { + return timestamp.compareTo(other.timestamp); + } + + /** + * Determines whether two LogEntries refer to the same event. * @param other some object to compare to * @return (see above) * @since 0.33 @@ -7180,28 +6387,12 @@ public String toString() @Override public boolean equals(Object other) { - if (!super.equals(other)) - return false; if (!(other instanceof LogEntry)) return false; LogEntry le = (LogEntry)other; - return Objects.equals(type, le.type) - && Objects.equals(action, le.action); - } - - /** - * Computes a hashcode for this LogEntry based on the underlying - * {@linkplain Event#hashCode() Event}, type and action. - * @return (see above) - * @since 0.35 - */ - @Override - public int hashCode() - { - int hc = super.hashCode(); - hc = 127 * hc + type.hashCode(); - hc = 127 * hc + Objects.hashCode(action); - return hc; + return type.equals(le.type) && action.equals(le.action) && + user.equals(le.user) && target.equals(le.target) && + reason.equals(le.reason) && timestamp.equals(le.timestamp); } } @@ -7209,39 +6400,46 @@ public int hashCode() * Represents a contribution and/or a revision to a page. * @since 0.17 */ - public class Revision extends Event + public class Revision implements Comparable { - private final boolean minor, bot, rvnew; - private final String sha1; - private long rcid = -1; + private boolean minor, bot, rvnew; + private String summary; + private long revid, rcid = -1; private long previous = 0, next = 0; - private int size = 0, sizediff = 0; + private OffsetDateTime timestamp; + private String user; + private String title; + private String rollbacktoken = null; + private int size = 0; + private int sizediff = 0; + private boolean summaryDeleted = false, userDeleted = false, contentDeleted = false; private boolean pageDeleted = false; /** * Constructs a new Revision object. * @param revid the id of the revision (this is a long since - * {{NUMBEROFEDITS}} on en.wikipedia.org is now (January 2018) ~38% - * of {@code Integer.MAX_VALUE} + * {{NUMBEROFEDITS}} on en.wikipedia.org is now (January 2012) ~25% + * of Integer.MAX_VALUE * @param timestamp when this revision was made - * @param user the user making this revision (may be anonymous) - * @param comment the edit summary - * @param parsedcomment the edit summary, parsed into HTML * @param title the concerned article - * @param sha1 the SHA-1 hash of the revision + * @param summary the edit summary + * @param user the user making this revision (may be anonymous, if not + * use User.getUsername()) * @param minor whether this was a minor edit * @param bot whether this was a bot edit * @param rvnew whether this revision created a new page * @param size the size of the revision * @since 0.17 */ - public Revision(long revid, OffsetDateTime timestamp, String user, String comment, - String parsedcomment, String title, String sha1, boolean minor, boolean bot, - boolean rvnew, int size) + public Revision(long revid, OffsetDateTime timestamp, String title, String summary, String user, + boolean minor, boolean bot, boolean rvnew, int size) { - super(revid, timestamp, user, Objects.requireNonNull(title), comment, parsedcomment); - this.sha1 = sha1; + this.revid = revid; + this.timestamp = timestamp; + this.summary = summary; this.minor = minor; + this.user = user; + this.title = title; this.bot = bot; this.rvnew = rvnew; this.size = size; @@ -7249,7 +6447,7 @@ public Revision(long revid, OffsetDateTime timestamp, String user, String commen /** * Fetches the contents of this revision. - * @return the contents of the appropriate article at timestamp + * @return the contents of the appropriate article at timestamp * @throws IOException if a network error occurs * @throws IllegalArgumentException if page == Special:Log/xxx. * @since 0.17 @@ -7257,125 +6455,176 @@ public Revision(long revid, OffsetDateTime timestamp, String user, String commen public String getText() throws IOException { // logs have no content - if (getID() < 1L) + if (revid < 1L) throw new IllegalArgumentException("Log entries have no valid content!"); // TODO: returning a 404 here when revision content has been deleted // is not a good idea. if (pageDeleted) // FIXME: broken if a page is live, but has deleted revisions { - Map getparams = new HashMap<>(); - getparams.put("action", "query"); - getparams.put("prop", "deletedrevisions"); - getparams.put("drvprop", "content"); - getparams.put("revids", String.valueOf(getID())); - String temp = makeApiCall(getparams, null, "Revision.getText"); - detectUncheckedErrors(temp, null, null); + String url = query + "prop=deletedrevisions&drvprop=content&revids=" + revid; + String temp = fetch(url, "Revision.getText"); int a = temp.indexOf("', a) + 1; + a = temp.indexOf(">", a) + 1; int b = temp.indexOf("", a); // tag not present if revision has no content - log(Level.INFO, "Revision.getText", "Successfully retrieved text of revision " + getID()); + log(Level.INFO, "Revision.getText", "Successfully retrieved text of revision " + revid); return (b < 0) ? "" : temp.substring(a, b); } else - return Wiki.this.getText(null, new long[] { getID() }, -1).get(0); + { + String url = base + encode(title, true) + "&oldid=" + revid + "&action=raw"; + String temp = fetch(url, "Revision.getText"); + log(Level.INFO, "Revision.getText", "Successfully retrieved text of revision " + revid); + return temp; + } } /** * Gets the rendered text of this revision. * @return the rendered contents of the appropriate article at - * timestamp + * timestamp * @throws IOException if a network error occurs * @throws IllegalArgumentException if page == Special:Log/xxx. * @since 0.17 */ public String getRenderedText() throws IOException { - if (getID() < 1L) + // logs have no content + if (revid < 1L) throw new IllegalArgumentException("Log entries have no valid content!"); - Map content = new HashMap<>(); - content.put("revision", this); - return Wiki.this.parse(content, -1, false); + + String temp; + if (pageDeleted) + { + String url = query + "prop=deletedrevisions&drvprop=content&drvparse=1&revids=" + revid; + temp = fetch(url, "Revision.getRenderedText"); + // TODO + } + else + { + String url = base + "&action=render&oldid=" + revid; + temp = fetch(url, "Revision.getRenderedText"); + } + log(Level.INFO, "Revision.getRenderedText", "Successfully retrieved rendered text of revision " + revid); + return decode(temp); } /** - * Returns the SHA-1 hash (base 16, lower case) of the content of this - * revision, or {@code null} if the revision content is RevisionDeleted - * and we cannot access it. - * - *

Warnings: - *

    - *
  • Not available through {@link #watchlist(RequestHelper)} or {@link - * #contribs(List, String, RequestHelper)}. - *
- * + * Returns true if the revision content is RevisionDeleted. * @return (see above) - * @since 0.35 + * @since 0.31 */ - public String getSha1() + public boolean isContentDeleted() { - return sha1; + return contentDeleted; } /** - * Returns a HTML rendered diff table of this revision to other. - * See {@link #diff(Map, Map)} for full documentation. - * + * Returns a HTML rendered diff table; see the table at the example. + * Returns null for page creations, moves, protections and similar + * dummy edits ( + * example). + * * @param other another revision on the same page. * @return the difference between this and the other revision * @throws IOException if a network error occurs - * @throws SecurityException if this or the other revision is - * RevisionDeleted and the user lacks the necessary privileges * @since 0.21 */ public String diff(Revision other) throws IOException { - return Wiki.this.diff(Map.of("revision", this), Map.of("revision", other)); + return diff(other.revid, ""); } /** - * Returns a HTML rendered diff table from this revision to the given - * text. Useful for emulating the "show changes" - * functionality. See the table at the example. - * * @param text some wikitext * @return the difference between this and the the text provided * @throws IOException if a network error occurs - * @throws SecurityException if this or the other revision is - * RevisionDeleted and the user lacks the necessary privileges * @since 0.21 */ public String diff(String text) throws IOException { - return Wiki.this.diff(Map.of("revision", this), Map.of("text", text)); + return diff(0L, text); } /** - * Returns a HTML rendered diff table from this revision to the given - * oldid. See {@link #diff(Map, Map)} for full documentation. - * - * @param oldid the oldid of a revision on the same page. {@link - * Wiki#NEXT_REVISION}, {@link Wiki#PREVIOUS_REVISION} and {@link - * Wiki#CURRENT_REVISION} can be used here for obvious effect. + * Returns a HTML rendered diff table; see the table at the example. + * Returns null for page creations, moves, protections and + * similar dummy edits (example) + * and pairs of revisions where there is no difference. + * + * @param oldid the oldid of a revision on the same page. {@link Wiki#NEXT_REVISION}, + * {@link Wiki#PREVIOUS_REVISION} and {@link Wiki#CURRENT_REVISION} can + * be used here for obvious effect. * @return the difference between this and the other revision * @throws IOException if a network error occurs - * @throws SecurityException if this or the other revision is - * RevisionDeleted and the user lacks the necessary privileges * @since 0.26 */ public String diff(long oldid) throws IOException { - Map from = new HashMap<>(); - from.put("revision", this); - Map to = new HashMap<>(); - to.put("revid", oldid); - return Wiki.this.diff(from, to); + return diff(oldid, ""); + } + + /** + * Fetches a HTML rendered diff table; see the table at the example. + * Returns null for page creations, moves, protections and similar + * dummy edits ( + * example) and pairs of revisions where there is no difference. + * + * @param oldid the id of another revision; (exclusive) or + * @param text some wikitext to compare against + * @return a difference between oldid or text or null if there is no + * diff. + * @throws IOException if a network error occurs + * @since 0.21 + */ + protected String diff(long oldid, String text) throws IOException + { + StringBuilder temp = new StringBuilder(); + if (oldid == NEXT_REVISION) + temp.append("&torelative=next"); + else if (oldid == CURRENT_REVISION) + temp.append("&torelative=cur"); + else if (oldid == PREVIOUS_REVISION) + temp.append("&torelative=prev"); + else if (oldid == 0L) + { + temp.append("&totext="); + temp.append(text); + } + else + { + temp.append("&torev="); + temp.append(oldid); + } + + String line = post(apiUrl + "action=compare&fromrev=" + getRevid(), temp.toString(), "Revision.diff"); + + // strip extraneous information + if (line.contains("")) + { + int a = line.indexOf("", a) + 1; + int b = line.indexOf("", a); + return decode(line.substring(a, b)); + } + else + // tag has no content if there is no diff or the two + // revisions are identical. In particular, the API does not + // distinguish between: + // https://en.wikipedia.org/w/index.php?title=Source_Filmmaker&diff=804972897&oldid=803731343 (no difference) + // https://en.wikipedia.org/w/index.php?title=Dayo_Israel&oldid=738178354&diff=prev (dummy edit) + return null; } /** - * Determines whether this Revision is equal to another based on the - * underlying {@linkplain Event#equals(Object) Event}. + * Determines whether this Revision is equal to another object. * @param o an object * @return whether o is equal to this object * @since 0.17 @@ -7383,29 +6632,20 @@ public String diff(long oldid) throws IOException @Override public boolean equals(Object o) { - // Note to self: don't use SHA-1 until all API calls provide it. - if (!super.equals(o)) - return false; if (!(o instanceof Revision)) return false; - // Revision rev = (Revision)o; - // return Objects.equals(sha1, rev.sha1); - return true; + return revid == ((Revision)o).revid; } /** - * Returns a hash code of this revision based on the underlying - * {@linkplain Event#hashCode() Event}. + * Returns a hash code of this revision. * @return a hash code * @since 0.17 */ @Override public int hashCode() { - // Note to self: don't use SHA-1 until all API calls provide it. - int hc = super.hashCode(); - hc = 127 * hc; - return hc; + return (int)revid * 2 - Wiki.this.hashCode(); } /** @@ -7431,15 +6671,12 @@ public boolean isBot() } /** - * Determines whether this revision created a new page. - * - *

Warnings: - *

    - *
  • Returning {@code true} does not imply this is the bottommost - * revision on the page due to histmerges. - *
  • Not available through {@link #getPageHistory(String, Wiki.RequestHelper)} - *
- * + * Determines whether this revision created a new page.
+ * WARNING: Will return false for all revisions prior to 2007 + * (I think?) -- this is a MediaWiki problem.
+ * WARNING: Returning true does not imply this is the bottommost + * revision on the page due to histmerges.
+ * WARNING: Not accessible through getPageHistory() -- a MW problem. * @return (see above) * @since 0.27 */ @@ -7449,8 +6686,51 @@ public boolean isNew() } /** - * Returns {@code true} if this revision is deleted (not the same as - * RevisionDeleted). + * Returns the edit summary for this revision, or null if the summary + * was RevisionDeleted and you lack the necessary privileges. + * @return the edit summary + * @since 0.17 + */ + public String getSummary() + { + return summary; + } + + /** + * Returns true if the edit summary is RevisionDeleted. + * @return (see above) + * @since 0.30 + */ + public boolean isSummaryDeleted() + { + return summaryDeleted; + } + + /** + * Returns the user or anon who created this revision. You should + * pass this (if not an IP) to getUser(String) to obtain a + * User object. Returns null if the user was RevisionDeleted and you + * lack the necessary privileges. + * @return the user or anon + * @since 0.17 + */ + public String getUser() + { + return user; + } + + /** + * Returns true if the user is RevisionDeleted. + * @return (see above) + * @since 0.30 + */ + public boolean isUserDeleted() + { + return userDeleted; + } + + /** + * Returns true if this revision is deleted (different from revdeleted). * @return (see above) * @since 0.31 */ @@ -7459,6 +6739,37 @@ public boolean isPageDeleted() return pageDeleted; } + /** + * Returns the page to which this revision was made. + * @return the page + * @since 0.17 + */ + public String getPage() + { + return title; + } + + /** + * Returns the oldid of this revision. Don't confuse this with + * rcid + * @return the oldid (long) + * @since 0.17 + */ + public long getRevid() + { + return revid; + } + + /** + * Gets the time that this revision was made. + * @return the timestamp + * @since 0.17 + */ + public OffsetDateTime getTimestamp() + { + return timestamp; + } + /** * Gets the size of this revision in bytes. * @return see above @@ -7470,8 +6781,7 @@ public int getSize() } /** - * Returns the change in page size caused by this revision. Not available - * through getPageHistory or getDeletedHistory. + * Returns the change in page size caused by this revision. * @return see above * @since 0.28 */ @@ -7488,8 +6798,22 @@ public int getSizeDiff() @Override public String toString() { - StringBuilder sb = new StringBuilder(super.toString()); - sb.deleteCharAt(sb.length() - 1); + StringBuilder sb = new StringBuilder("Revision[oldid="); + sb.append(revid); + sb.append(",page=\""); + sb.append(title); + sb.append("\",user="); + sb.append(user == null ? "[hidden]" : user); + sb.append(",userdeleted="); + sb.append(userDeleted); + sb.append(",timestamp="); + sb.append(timestamp.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + sb.append(",summary=\""); + sb.append(summary == null ? "[hidden]" : summary); + sb.append("\",summarydeleted="); + sb.append(summaryDeleted); + sb.append(",contentDeleted="); + sb.append(contentDeleted); sb.append(",minor="); sb.append(minor); sb.append(",bot="); @@ -7502,10 +6826,25 @@ public String toString() sb.append(previous); sb.append(",next="); sb.append(next); + sb.append(",rollbacktoken="); + sb.append(rollbacktoken == null ? "null" : rollbacktoken); sb.append("]"); return sb.toString(); } + /** + * Compares this revision to another revision based on the recentness + * of their timestamps. + * @param other the revision to compare + * @return whether this object is equal to + * @since 0.18 + */ + @Override + public int compareTo(Wiki.Revision other) + { + return timestamp.compareTo(timestamp); + } + /** * Gets the previous revision. * @return the previous revision, or null if this is the first revision @@ -7553,615 +6892,211 @@ public long getRcid() } /** - * Gets a permanent URL to the human-readable version of this Revision. - * This URL uses index.php, not Special:Permanentlink for ease of adding - * other parameters. - * @return (see above) - * @since 0.35 + * Sets a rollback token for this revision. + * @param token a rollback token + * @since 0.24 */ - public String permanentUrl() + public void setRollbackToken(String token) { - return getIndexPhpUrl() + "?oldid=" + getID(); + rollbacktoken = token; } /** - * Reverts this revision using the rollback method. - * - * @throws IOException if a network error occurs - * @throws SecurityException if the user lacks the privileges to rollback - * @throws CredentialExpiredException if cookies have expired - * @throws AccountLockedException if the user is blocked - * @see Wiki#rollback(org.wikipedia.Wiki.Revision) - * @since 0.19 + * Gets the rollback token for this revision. Can be null, and often + * for good reasons: cannot rollback or not top revision. + * @return the rollback token + * @since 0.24 */ - public void rollback() throws IOException, LoginException + public String getRollbackToken() { - Wiki.this.rollback(this, false, ""); + return rollbacktoken; } /** - * Reverts this revision using the rollback method. - * - * @param bot mark this and the reverted revision(s) as bot edits - * @param reason (optional) a custom reason + * Reverts this revision using the rollback method. See + * {@link Wiki#rollback(org.wikipedia.Wiki.Revision)}. + * * @throws IOException if a network error occurs - * @throws SecurityException if the user lacks the privileges to rollback + * @throws CredentialNotFoundException if not logged in or user is not + * an admin * @throws CredentialExpiredException if cookies have expired * @throws AccountLockedException if the user is blocked - * @see Wiki#rollback(org.wikipedia.Wiki.Revision) * @since 0.19 */ - public void rollback(boolean bot, String reason) throws IOException, LoginException - { - Wiki.this.rollback(this, bot, reason); - } - } - - /** - * Vehicle for stuffing standard optional parameters into Wiki queries. - * {@code RequestHelper} objects are reusable. The following example - * fetches articles from the back of the new pages queue on the - * English Wikipedia. - * - * {@code 
-     *  Wiki.RequestHelper rh = enWiki.new RequestHelper()
-     *      .inNamespaces(Wiki.MAIN_NAMESPACE)
-     *      .reverse();
-     *  List newpages = enWiki.newPages(rh);
-     *
} - * - * @since 0.36 - */ - public class RequestHelper - { - private String title; - private String byuser; - private OffsetDateTime earliest, latest; - private int[] localns = new int[0]; - private boolean reverse = false; - private String notbyuser; - private String tag; - private Map options; - private String requestType; - private int limit = -1; - - /** - * Creates a new RequestHelper. - */ - public RequestHelper() - { - } - - /** - * Limits query results to Events occuring on the given title. If a - * query mandates a title parameter (e.g. {@link #getPageHistory(String, - * RequestHelper)}, don't use this. Use the parameter in the query - * method instead. - * @param title a page title - * @return this RequestHelper - */ - public RequestHelper byTitle(String title) - { - this.title = (title == null) ? null : normalize(title); - return this; - } - - /** - * Limits query results to Events triggered by the given user. If a query - * mandates a user parameter (e.g. {@link #contribs(List, String, RequestHelper)}, - * don't use this. Use the parameter in the query method instead. - * @param byuser some username or IP address - * @return this RequestHelper - */ - public RequestHelper byUser(String byuser) - { - this.byuser = (byuser == null) ? null : normalize(byuser); - return this; - } - - /** - * Limit results to be within this date range. - * @param earliest the lower (earliest) date bound, use {@code null} to - * not set one - * @param latest the higher (latest) date bound, use {@code null} to - * not set one - * @throws IllegalArgumentException if {@code earliest.isAfter(latest)} - * @return this RequestHelper - */ - public RequestHelper withinDateRange(OffsetDateTime earliest, OffsetDateTime latest) - { - if (earliest != null && latest != null && earliest.isAfter(latest)) - throw new IllegalArgumentException("Earliest date must be before latest date!"); - this.earliest = earliest; - this.latest = latest; - return this; - } - - /** - * Limits query results to the given namespaces. - * @param ns a list of namespaces - * @return this RequestHelper - */ - public RequestHelper inNamespaces(int... ns) - { - localns = ns; - return this; - } - - /** - * Should we perform this query in reverse order (earliest first). - * @param reverse whether to reverse this query - * @return this RequestHelper - */ - public RequestHelper reverse(boolean reverse) - { - this.reverse = reverse; - return this; - } - - /** - * Limits query results to {@link Event Events} that have been tagged - * with the given tag. - * @param tag a change tag - * @return this RequestHelper - */ - public RequestHelper taggedWith(String tag) - { - this.tag = tag; - return this; - } - - /** - * Limits query results to Events NOT triggered by the given user. - * @param notbyuser some username or IP address to exclude - * @return this RequestHelper - */ - public RequestHelper notByUser(String notbyuser) - { - this.notbyuser = (notbyuser == null) ? null : normalize(notbyuser); - return this; - } - - /** - * Return no more than the given number of results. Overrides {@linkplain - * #setQueryLimit(int) global limits}. Supply a negative integer to use - * global limits. - * @param limit a positive integer - * @return this RequestHelper - */ - public RequestHelper limitedTo(int limit) - { - this.limit = limit; - return this; - } - - /** - * Filters a set of returned results using the given options. Please - * check calling method documentation for supported options. - * - *

- * When filtering revisions, available keys may include "minor", "top", - * "new", "bot", "anon", "redirect", "patrolled" and "unread" for - * vanilla MediaWiki. Extensions may define their own. For instance, - * {@code rcoptions = { minor = true, anon = false, patrolled = false}} - * returns all minor edits from logged in users that aren't patrolled. - * Setting "patrolled" limits results to no older than retention in - * the recentchanges - * table. - * - * @param options the options to filter by - * @return this RequestHelper - */ - public RequestHelper filterBy(Map options) - { - this.options = options; - return this; - } - - /** - * Sets the prefix of API request parameters (the XX in XXlimit, XXdir, - * XXnamespace and so forth). Internal use only. - * @param prefix the prefix to use (must not be null) - */ - protected void setRequestType(String prefix) - { - requestType = Objects.requireNonNull(prefix); - } - - /** - * Returns a HTTP request parameter containing the title to get - * events for, or an empty map if not wanted. - * @return (see above) - */ - protected Map addTitleParameter() - { - if (title != null) - return Map.of(requestType + "title", title); - return Collections.emptyMap(); - } - - /** - * Returns a HTTP request parameter containing the user to filter - * returned events by, or an empty map if not wanted. - * @return (see above) - */ - protected Map addUserParameter() - { - if (byuser != null) - return Map.of(requestType + "user", byuser); - return Collections.emptyMap(); - } - - /** - * Returns a HTTP request parameter containing the dates to start - * and end enumeration, or an empty map if not wanted. - * @return (see above) - */ - protected Map addDateRangeParameters() - { - // https://phabricator.wikimedia.org/T16449 - Map temp = new HashMap<>(); - OffsetDateTime odt = reverse ? earliest : latest; - if (odt != null) - temp.put(requestType + "start", - // https://www.mediawiki.org/wiki/Timestamp - // https://github.com/MER-C/wiki-java/issues/170 - odt.withOffsetSameInstant(ZoneOffset.UTC) - .truncatedTo(ChronoUnit.MICROS) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - odt = reverse ? latest : earliest; - if (odt != null) - temp.put(requestType + "end", - // https://www.mediawiki.org/wiki/Timestamp - // https://github.com/MER-C/wiki-java/issues/170 - odt.withOffsetSameInstant(ZoneOffset.UTC) - .truncatedTo(ChronoUnit.MICROS) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - return temp; - } - - /** - * Returns a HTTP request parameter containing the namespaces to limit - * this query to, or an empty map if not wanted. - * @return (see above) - */ - protected Map addNamespaceParameter() - { - if (localns.length != 0) - return Map.of(requestType + "namespace", constructNamespaceString(localns)); - return Collections.emptyMap(); - } - - /** - * Returns a HTTP request parameter instructing the API to reverse the - * query, or an empty map if not wanted. - * @return (see above) - */ - protected Map addReverseParameter() - { - return Map.of(requestType + "dir", reverse ? "newer" : "older"); - } - - /** - * Returns a HTTP request parameter containing the tag to limit - * returned events to, or an empty map if not wanted. - * @return (see above) - */ - protected Map addTagParameter() - { - if (tag != null) - return Map.of(requestType + "tag", tag); - return Collections.emptyMap(); - } - - /** - * Returns a HTTP request parameter containing the user to exclude - * when returning events, or an empty map if not wanted. - * @return (see above) - */ - protected Map addExcludeUserParameter() - { - if (notbyuser != null) - return Map.of(requestType + "excludeuser", notbyuser); - return Collections.emptyMap(); - } - - /** - * Returns HTTP request parameter(s) containing flags to filter returned - * revisions by, or an empty map if not wanted. - * @return (see above) - */ - protected Map addShowParameter() + public void rollback() throws IOException, LoginException { - Map temp = new HashMap<>(); - if (options != null && !options.isEmpty()) - { - // deal with MW API annoyance for action=query&list=watchlist - see watchlist(rh) - Boolean top = null; - if (requestType.equals("wl")) - { - top = options.remove("top"); - if (Boolean.TRUE.equals(top)) - temp.put("wlallrev", "1"); - } - - StringBuilder sb = new StringBuilder(); - options.forEach((key, value) -> - { - if (Boolean.FALSE.equals(value)) - sb.append('!'); - sb.append(key); - sb.append("|"); - }); - temp.put(requestType + "show", sb.substring(0, sb.length() - 1)); - - if (top != null) // put it back - options.put("top", top); - } - return temp; + Wiki.this.rollback(this, false, ""); } /** - * Returns the number of results the query should be limited to. If not - * present, use {@linkplain #setQueryLimit(int) global limits}. - * @return (see above) + * Reverts this revision using the rollback method. See + * {@link Wiki#rollback(org.wikipedia.Wiki.Revision)}. + * + * @param bot mark this and the reverted revision(s) as bot edits + * @param reason (optional) a custom reason + * @throws IOException if a network error occurs + * @throws CredentialNotFoundException if not logged in or user is not + * an admin + * @throws CredentialExpiredException if cookies have expired + * @throws AccountLockedException if the user is blocked + * @since 0.19 */ - public int limit() + public void rollback(boolean bot, String reason) throws IOException, LoginException { - return limit; + Wiki.this.rollback(this, bot, reason); } } // INTERNALS - /** - * Performs a vectorized action=query&prop=X type API query - * over titles. - * @param queryPrefix the request type prefix (e.g. "pl" for prop=links) - * @param getparams a bunch of parameters to send via HTTP GET - * @param titles a list of titles - * @param caller the name of the calling method - * @param limit fetch no more than this many results - * @param parser a BiConsumer that parses the XML returned by the MediaWiki - * API into things we want, dumping them into the given List - * @return a list of results, where each element corresponds to the element - * at the same index in the input title list - * @since 0.36 - * @throws IOException if a network error occurs - */ - protected List> makeVectorizedQuery(String queryPrefix, Map getparams, - List titles, String caller, int limit, BiConsumer> parser) throws IOException - { - // copy because normalization and redirect resolvers overwrite - List titles2 = new ArrayList<>(titles); - List>> stuff = new ArrayList<>(); - Map postparams = new HashMap<>(); - for (String temp : constructTitleString(titles2)) - { - postparams.put("titles", temp); - stuff.addAll(makeListQuery(queryPrefix, getparams, postparams, caller, -1, (line, results) -> - { - // Split the result into individual listings for each article. - String[] x = line.split(" list = new ArrayList<>(); - parser.accept(x[i], list); - - Map> intermediate = new HashMap<>(); - intermediate.put(parsedtitle, list); - results.add(intermediate); - } - })); - } - - // prepare the return list - List> ret = Stream.generate(() -> new ArrayList()) - .limit(titles2.size()) - .collect(Collectors.toCollection(ArrayList::new)); - // then retrieve the results from the intermediate list of maps, - // ensuring results correspond to inputs - stuff.forEach(map -> - { - String parsedtitle = map.keySet().iterator().next(); - List templates = map.get(parsedtitle); - for (int i = 0; i < titles2.size(); i++) - if (titles2.get(i).equals(parsedtitle)) - ret.get(i).addAll(templates); - }); - return ret; - } - /** * Fetches list-type results from the MediaWiki API. - * - * @param a class describing the parsed API results (e.g. String, + * + * @param a class describing the parsed API results (e.g. String, * LogEntry, Revision) * @param queryPrefix the request type prefix (e.g. "pl" for prop=links) - * @param getparams a bunch of parameters to send via HTTP GET - * @param postparams if not null, send these parameters via POST (see - * {@link #makeApiCall(Map, Map, String) }). + * @param url the query URL, without the limit and XXcontinue parameters * @param caller the name of the calling method - * @param limit fetch no more than this many results * @param parser a BiConsumer that parses the XML returned by the MediaWiki * API into things we want, dumping them into the given List * @return the query results * @throws IOException if a network error occurs - * @throws SecurityException if we don't have the credentials to perform a - * privileged action (mostly avoidable) * @since 0.34 */ - protected List makeListQuery(String queryPrefix, Map getparams, - Map postparams, String caller, int limit, BiConsumer> parser) throws IOException + protected List queryAPIResult(String queryPrefix, StringBuilder url, String caller, + BiConsumer> parser) throws IOException { - if (limit < 0) - limit = querylimit; - getparams.put("action", "query"); List results = new ArrayList<>(1333); - String limitstring = queryPrefix + "limit"; + StringBuilder xxcontinue = new StringBuilder(); + url.append("&"); + url.append(queryPrefix); + url.append("limit="); do { - getparams.put(limitstring, String.valueOf(Math.min(limit - results.size(), max))); - String line = makeApiCall(getparams, postparams, caller); - detectUncheckedErrors(line, null, null); - getparams.keySet().removeIf(param -> param.endsWith("continue")); - + int limit = Math.min(querylimit - results.size(), max); + String tempurl = url.toString() + limit; + String line = fetch(tempurl + xxcontinue.toString(), caller); + xxcontinue.setLength(0); + // Continuation parameter has form: // if (line.contains("", a); - String cont = line.substring(a, b); - for (String contpair : cont.split("\" ")) - { - contpair = " " + contpair.trim(); - String contattr = contpair.substring(0, contpair.indexOf("=\"")); - getparams.put(contattr.trim(), parseAttribute(cont, contattr, 0)); - } + int a = line.indexOf("", a); + String[] temp = line.substring(a, b).split("\""); + xxcontinue.append("&"); + xxcontinue.append(temp[0]); + xxcontinue.append(encode(temp[1], false)); + xxcontinue.append(temp[2].replace(" ", "&")); + xxcontinue.append(encode(temp[3], false)); } - + parser.accept(line, results); } - while (getparams.containsKey("continue") && results.size() < limit); + while (xxcontinue.length() != 0 && results.size() < querylimit); return results; } - - // miscellany + // miscellany + /** - * Convenience method for checking user permissions. - * @param right a user rights - * @param morerights additional user rights - * @throws SecurityException if the permission check fails - * @since 0.37 - */ - protected void checkPermissions(String action, String right, String... morerights) - { - if (user == null) - throw new SecurityException("Cannot " + action + ": not logged in."); - if (!user.isAllowedTo(right, morerights)) - throw new SecurityException("Cannot " + action + ": permission denied."); - } - - /** - * Constructs, sends and handles calls to the MediaWiki API. This is a - * low-level method for making your own, custom API calls. - * - *

- * If postparams is not {@code null} or empty, the request is - * sent using HTTP GET, otherwise it is sent using HTTP POST. A - * {@code byte[]} value in postparams causes the request to be - * sent as a multipart POST. Anything else is converted to String via the - * following means: - * - *

    - *
  • String[] -- {@code String.join("|", arr)} - *
  • StringBuilder -- {@code sb.toString()} - *
  • Number -- {@code num.toString()} - *
  • OffsetDateTime -- {@code date.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)} - *
  • {@code Collection} -- {@code coll.stream() - * .map(item -> convertToString(item)) // using the above rules - * .collect(Collectors.joining("|"))} - *
- * - *

- * All supplied Strings and objects converted to String are automatically - * URLEncoded in UTF-8 if this is a normal POST request. + * A generic URL content fetcher. This is only useful for GET requests, + * which is almost everything that doesn't modify the wiki. Might be + * useful for subclasses. * - *

* Here we also check the database lag and wait if it exceeds * maxlag, see * here for how this works. * - * @param getparams append these parameters to the urlbase - * @param postparams if null, send the request using POST otherwise use GET + * @param url the url to fetch * @param caller the caller of this method - * @return the server response + * @return the content of the fetched URL * @throws IOException if a network error occurs - * @throws SecurityException if we don't have the credentials to perform a - * privileged action (mostly avoidable) * @throws AssertionError if assert=user|bot fails - * @see Multipart/form-data * @since 0.18 */ - public String makeApiCall(Map getparams, Map postparams, String caller) throws IOException + protected String fetch(String url, String caller) throws IOException { - // build the URL - StringBuilder urlbuilder = new StringBuilder(apiUrl + "?"); - getparams.putAll(defaultApiParams); - for (Map.Entry entry : getparams.entrySet()) - { - urlbuilder.append('&'); - urlbuilder.append(entry.getKey()); - urlbuilder.append('='); - urlbuilder.append(URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8)); - } - String url = urlbuilder.toString(); - - // POST stuff - boolean isPOST = (postparams != null && !postparams.isEmpty()); - StringBuilder stringPostBody = new StringBuilder(); - boolean multipart = false; - ArrayList multipartPostBody = new ArrayList<>(); - String boundary = "----------NEXT PART----------"; - if (isPOST) + String temp = null; + int tries = maxtries; + do { - // determine whether this is a multipart post and convert any values - // to String if necessary - for (Map.Entry entry : postparams.entrySet()) + logurl(url, caller); + tries--; + try { - Object value = entry.getValue(); - if (value instanceof byte[]) - multipart = true; - else - entry.setValue(convertToString(value)); - } - - // now we know how we're sending it, construct the post body - if (multipart) - { - byte[] nextpart = ("--" + boundary + "\r\n\"Content-Disposition: form-data; name=\\\"\"") - .getBytes(StandardCharsets.UTF_8); - for (Map.Entry entry : postparams.entrySet()) + // connect + URLConnection connection = makeConnection(url); + connection.setConnectTimeout(CONNECTION_CONNECT_TIMEOUT_MSEC); + connection.setReadTimeout(CONNECTION_READ_TIMEOUT_MSEC); + setCookies(connection); + connection.connect(); + grabCookies(connection); + + // check lag and retry + if (checkLag(connection)) + return fetch(url, caller); + + // get the text + String line; + StringBuilder text = new StringBuilder(100000); + try (BufferedReader in = new BufferedReader(new InputStreamReader( + zipped ? new GZIPInputStream(connection.getInputStream()) : connection.getInputStream(), "UTF-8"))) { - multipartPostBody.add(nextpart); - Object value = entry.getValue(); - multipartPostBody.add((entry.getKey() + "\"\r\n").getBytes(StandardCharsets.UTF_8)); - if (value instanceof String) - multipartPostBody.add(("Content-Type: text/plain; charset=UTF-8\r\n\r\n" + (String)value + "\r\n") - .getBytes(StandardCharsets.UTF_8)); - else if (value instanceof byte[]) + while ((line = in.readLine()) != null) { - multipartPostBody.add("Content-Type: application/octet-stream\r\n\r\n".getBytes(StandardCharsets.UTF_8)); - multipartPostBody.add((byte[])value); - multipartPostBody.add("\r\n".getBytes(StandardCharsets.UTF_8)); + text.append(line); + text.append("\n"); } } - multipartPostBody.add((boundary + "--\r\n").getBytes(StandardCharsets.UTF_8)); + temp = text.toString(); } - else + catch (IOException ex) { - // automatically encode Strings sent via normal POST - for (Map.Entry entry : postparams.entrySet()) + if (tries == 0) + throw ex; + try + { + Thread.sleep(10000); + } + catch (InterruptedException ignored) { - stringPostBody.append('&'); - stringPostBody.append(entry.getKey()); - stringPostBody.append('='); - stringPostBody.append(URLEncoder.encode(entry.getValue().toString(), StandardCharsets.UTF_8)); } } } + while (temp == null); + if (temp.contains(" hr = client.send(connection.build(), HttpResponse.BodyHandlers.ofInputStream()); - boolean zipped_ = hr.headers().firstValue("Content-Encoding").orElse("").equals("gzip"); - if (checkLag(hr)) + URLConnection connection = makeConnection(url); + setCookies(connection); + connection.setDoOutput(true); + connection.setConnectTimeout(CONNECTION_CONNECT_TIMEOUT_MSEC); + connection.setReadTimeout(CONNECTION_READ_TIMEOUT_MSEC); + connection.connect(); + + try (OutputStreamWriter out = new OutputStreamWriter(connection.getOutputStream(), "UTF-8")) { - tries++; - throw new HttpRetryException("Database lagged.", 503); + out.write(text); } + // check lag and retry + if (checkLag(connection)) + return post(url, text, caller); + grabCookies(connection); + StringBuilder buffer = new StringBuilder(100000); + String line; try (BufferedReader in = new BufferedReader(new InputStreamReader( - zipped_ ? new GZIPInputStream(hr.body()) : hr.body(), "UTF-8"))) + zipped ? new GZIPInputStream(connection.getInputStream()) : connection.getInputStream(), "UTF-8"))) { - response = in.lines().collect(Collectors.joining("\n")); + while ((line = in.readLine()) != null) + { + buffer.append(line); + buffer.append("\n"); + } } - - // Check for rate limit (though might be a long one e.g. email) - if (response.contains("error code=\"ratelimited\"")) + temp = buffer.toString(); + + // check for recoverable errors + + // rate limit (though might be a long one e.g. email) + if (temp.contains("error code=\"ratelimited\"")) { - // the Retry-After header field is useless here - // see https://phabricator.wikimedia.org/T172293 log(Level.WARNING, caller, "Server-side throttle hit."); - Thread.sleep(10000); throw new HttpRetryException("Action throttled.", 503); } - // Check for database lock - if (response.contains("error code=\"readonly\"")) + // database lock + if (temp.contains("error code=\"readonly\"")) { log(Level.WARNING, caller, "Database locked!"); - Thread.sleep(10000); throw new HttpRetryException("Database locked!", 503); } - - // No need to retry anymore, success or unrecoverable failure. - tries = 0; + return temp; } catch (IOException ex) { - // Exception deliberately ignored until retries are depleted. if (tries == 0) throw ex; - } - catch (InterruptedException ignored) - { + try + { + Thread.sleep(10000); + } + catch (InterruptedException ignored) + { + } } } - while (tries != 0); - - // empty response from server - if (response.isEmpty()) - throw new UnknownError("Received empty response from server!"); - return response; + while (temp.isEmpty()); + throw new AssertionError("Unreachable."); } /** - * Converts HTTP POST parameters to Strings. See {@link #makeApiCall(Map, - * Map, String)} for the description. - * @param param the parameter to convert - * @return that parameter, as a String - * @throws UnsupportedOperationException if param is not a supported data type - * @since 0.35 + * Performs a multi-part HTTP POST. + * @param url the url to post to + * @param params the POST parameters. Supported types: UTF-8 text, byte[]. + * Text and parameter names must NOT be URL encoded. + * @param caller the caller of this method + * @return the server response + * @throws IOException if a network error occurs + * @see #post(java.lang.String, java.lang.String, java.lang.String) + * @see Multipart/form-data + * @since 0.27 */ - private String convertToString(Object param) + protected String multipartPost(String url, Map params, String caller) throws IOException { - // TODO: Replace with type switch in JDK 11/12 - if (param instanceof String) - return (String)param; - else if (param instanceof StringBuilder || param instanceof Number) - return param.toString(); - else if (param instanceof String[]) - return String.join("|", (String[])param); - else if (param instanceof OffsetDateTime) - { - OffsetDateTime date = (OffsetDateTime)param; - // https://www.mediawiki.org/wiki/Timestamp - // https://github.com/MER-C/wiki-java/issues/170 - return date.atZoneSameInstant(ZoneOffset.UTC) - .truncatedTo(ChronoUnit.MICROS) - .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); - } - else if (param instanceof Collection) + String temp = ""; + int tries = maxtries; + do { - Collection coll = (Collection)param; - return coll.stream() - .map(item -> convertToString(item)) - .collect(Collectors.joining("|")); + logurl(url, caller); + tries--; + try + { + URLConnection connection = makeConnection(url); + String boundary = "----------NEXT PART----------"; + connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary); + setCookies(connection); + connection.setDoOutput(true); + connection.setConnectTimeout(CONNECTION_CONNECT_TIMEOUT_MSEC); + connection.setReadTimeout(CONNECTION_READ_TIMEOUT_MSEC); + connection.connect(); + + // write stuff to a local buffer + boundary = "--" + boundary + "\r\n"; + ByteArrayOutputStream bout = new ByteArrayOutputStream(); + try (DataOutputStream out = new DataOutputStream(bout)) + { + out.writeBytes(boundary); + + // write params + for (Map.Entry entry : params.entrySet()) + { + String name = entry.getKey(); + Object value = entry.getValue(); + out.writeBytes("Content-Disposition: form-data; name=\"" + name + "\"\r\n"); + if (value instanceof String) + { + out.writeBytes("Content-Type: text/plain; charset=UTF-8\r\n\r\n"); + out.write(((String)value).getBytes("UTF-8")); + } + else if (value instanceof byte[]) + { + out.writeBytes("Content-Type: application/octet-stream\r\n\r\n"); + out.write((byte[])value); + } + else + throw new UnsupportedOperationException("Unrecognized data type"); + out.writeBytes("\r\n"); + out.writeBytes(boundary); + } + out.writeBytes("--\r\n"); + } + try (OutputStream uout = connection.getOutputStream()) + { + // write the buffer to the URLConnection + uout.write(bout.toByteArray()); + } + + // check lag and retry + if (checkLag(connection)) + return multipartPost(url, params, caller); + + // done, read the response + grabCookies(connection); + String line; + StringBuilder buffer = new StringBuilder(100000); + try (BufferedReader in = new BufferedReader(new InputStreamReader( + zipped ? new GZIPInputStream(connection.getInputStream()) : connection.getInputStream(), "UTF-8"))) + { + while ((line = in.readLine()) != null) + { + buffer.append(line); + buffer.append("\n"); + } + } + temp = buffer.toString(); + + // check for recoverable errors + + // rate limit (though might be a long one e.g. email) + if (temp.contains("error code=\"ratelimited\"")) + { + log(Level.WARNING, caller, "Server-side throttle hit."); + throw new HttpRetryException("Action throttled.", 503); + } + // database lock + if (temp.contains("error code=\"readonly\"")) + { + log(Level.WARNING, caller, "Database locked!"); + throw new HttpRetryException("Database locked!", 503); + } + return temp; + } + catch (IOException ex) + { + if (tries == 0) + throw ex; + try + { + Thread.sleep(10000); + } + catch (InterruptedException ignored) + { + } + } } - else - throw new UnsupportedOperationException("Unrecognized data type"); + while (temp.isEmpty()); + throw new AssertionError("Unreachable."); } - + /** - * Checks for database lag and sleeps if {@code lag < getMaxLag()}. - * @param response the HTTP response received + * Checks for database lag and sleeps if lag > maxlag. + * @param connection the URL connection used in the request * @return true if there was sufficient database lag. - * @throws InterruptedException if any wait was interrupted - * @see #getMaxLag() - * @see - * MediaWiki documentation * @since 0.32 */ - protected synchronized boolean checkLag(HttpResponse response) throws InterruptedException + protected synchronized boolean checkLag(URLConnection connection) { - HttpHeaders hdrs = response.headers(); - long lag = hdrs.firstValueAsLong("X-Database-Lag").orElse(-5); + int lag = connection.getHeaderFieldInt("X-Database-Lag", -5); // X-Database-Lag is the current lag rounded down to the nearest integer. // Thus, we need to retry in case of equality. if (lag >= maxlag) { - long time = hdrs.firstValueAsLong("Retry-After").orElse(10); - logger.log(Level.WARNING, "Current database lag {0} s exceeds maxlag of {1} s, waiting {2} s.", new Object[] { lag, maxlag, time }); - Thread.sleep(time * 1000L); + try + { + int time = connection.getHeaderFieldInt("Retry-After", 10); + logger.log(Level.WARNING, "Current database lag {0} s exceeds maxlag of {1} s, waiting {2} s.", new Object[] { lag, maxlag, time }); + Thread.sleep(time * 1000); + } + catch (InterruptedException ignored) + { + } return true; } return false; } /** - * Sets the HTTPClient used by this instance. Use this to set a proxy, - * SSL parameters and the connection timeout. - * @param builder a HTTP request builder - * @since 0.37 - */ - public void setHttpClient(HttpClient.Builder builder) - { - client = builder.cookieHandler(cookies).build(); - } - - /** - * Creates a new HTTP request. Override to change request properties. + * Creates a new URL connection. Override to change SSL handling, use a + * proxy, etc. * @param url a URL string - * @return a HTTP request builder for that URL + * @return a connection to that URL * @throws IOException if a network error occurs * @since 0.31 */ - protected HttpRequest.Builder makeConnection(String url) throws IOException + protected URLConnection makeConnection(String url) throws IOException { - return HttpRequest.newBuilder(URI.create(url)) - .timeout(Duration.ofMillis(read_timeout_msec)) - .header("User-Agent", useragent) - .header("Accept-encoding", "gzip"); + return new URL(url).openConnection(); } - /** - * Checks for errors from standard read/write requests and throws the - * appropriate unchecked exception. - * - * @param response the response from the server to analyze - * @param errors additional errors to check for where throwing an unchecked - * exception is the desired behavior (function is of MediaWiki error - * message) - * @param warnings additional errors to check for where throwing an exception - * is not required (function is of MediaWiki error message) - * @throws RuntimeException or subclasses depending on the type of errors - * checked for - * @return whether the action was successful - * @since 0.37 - */ - protected boolean detectUncheckedErrors(String response, Map> errors, - Map> warnings) - { - if (response.contains("> uncheckederrors, - Map> info) throws IOException, LoginException + protected void checkErrorsAndUpdateStatus(String line, String caller) throws IOException, LoginException { // perform various status checks every 100 or so edits if (statuscounter > statusinterval) { // purge user rights in case of desysop or loss of other priviliges - user = getUsers(List.of(user.getUsername())).get(0); + user.getUserInfo(); if ((assertion & ASSERT_SYSOP) == ASSERT_SYSOP && !user.isA("sysop")) // assert user.isA("sysop") : "Sysop privileges missing or revoked, or session expired"; throw new AssertionError("Sysop privileges missing or revoked, or session expired"); @@ -8408,36 +7358,35 @@ protected boolean checkErrorsAndUpdateStatus(String line, String caller, else statuscounter++; - if (!line.contains(" constructRevisionString(long[] ids) + protected String[] constructRevisionString(long[] ids) { // sort and remove duplicates per https://mediawiki.org/wiki/API String[] sortedids = Arrays.stream(ids) .distinct() - .filter(id -> id >= 0) .sorted() .mapToObj(String::valueOf) .toArray(String[]::new); - + StringBuilder buffer = new StringBuilder(); - List chunks = new ArrayList<>(); + ArrayList chunks = new ArrayList<>(); for (int i = 0; i < sortedids.length; i++) { buffer.append(sortedids[i]); - if (i == ids.length - 1 || (i % slowmax) == slowmax - 1) + if (i == ids.length - 1 || i == slowmax - 1) { chunks.add(buffer.toString()); buffer.setLength(0); } else - buffer.append('|'); + buffer.append("%7C"); } - return chunks; + return chunks.toArray(new String[chunks.size()]); } /** * Cuts up a list of titles into batches for prop=X&titles=Y type queries. + * @param lengthBaseUrl the length of the url when no titles are present * @param titles a list of titles. + * @param limit whether to apply the maximum URL size * @return the titles ready for insertion into a URL + * @throws IOException if a network error occurs * @since 0.29 */ - protected List constructTitleString(List titles) + protected String[] constructTitleString(int lengthBaseUrl, String[] titles, boolean limit) throws IOException { - // sort and remove duplicates - List titles_unique = titles.stream().sorted().distinct().collect(Collectors.toList()); - + // sort and remove duplicates per https://mediawiki.org/wiki/API + Set set = new TreeSet<>(); + for (String title : titles) + set.add(encode(title, true)); + String[] titlesEnc = set.toArray(new String[set.size()]); + // actually construct the string + String titleStringToken = encode("|", false); ArrayList ret = new ArrayList<>(); - for (int i = 0; i < titles_unique.size() / slowmax + 1; i++) + StringBuilder buffer = new StringBuilder(); + buffer.append(titlesEnc[0]); + int num = 1; + for (int i = num; i < titlesEnc.length; i++) { - ret.add(String.join("|", - titles_unique.subList(i * slowmax, Math.min(titles_unique.size(), (i + 1) * slowmax)))); + if (num < slowmax && + buffer.length() + titleStringToken.length() + titlesEnc[i].length() < URL_LENGTH_LIMIT - lengthBaseUrl) + { + buffer.append(titleStringToken); + } + else + { + ret.add(buffer.toString()); + buffer.setLength(0); + num = 0; + } + buffer.append(titlesEnc[i]); + ++num; } - return ret; + ret.add(buffer.toString()); + return ret.toArray(new String[ret.size()]); + } + + /** + * UTF-8 encode the String with URLEncoder after optional normalization; + * Usually, normalize should be set to true when a title or name String is + * passed in as an argument of a method. + * + * @param text the text to encode + * @param normalize if the text should be normalized first + * @return the encoded text + * @throws IOException if a network error occurs during initialization of the namespaces + */ + private String encode(String text, boolean normalize) throws IOException + { + final String encoding = "UTF-8"; + if (normalize) + text = normalize(text); + return URLEncoder.encode(text, encoding); } /** * Convenience method for normalizing MediaWiki titles. (Converts all * underscores to spaces, localizes namespace names, fixes case of first - * char and does some other unicode fixes). Beware that this will not - * produce the same results as server-side normalization in a few corner - * cases, most notably: HTML entities and gender distinction in the user - * namespace prefix. + * char and does some other unicode fixes). * @param s the string to normalize * @return the normalized string * @throws IllegalArgumentException if the title is invalid - * @throws UncheckedIOException if the namespace cache has not been - * populated, and a network error occurs when populating it + * @throws IOException if a network error occurs * @since 0.27 */ - public String normalize(String s) + public String normalize(String s) throws IOException { - // remove section names - if (s.contains("#")) - s = s.substring(0, s.indexOf("#")); // remove leading colon if (s.startsWith(":")) s = s.substring(1); - s = s.replace('_', ' ').trim(); if (s.isEmpty()) - throw new IllegalArgumentException("Empty or whitespace only title."); + return s; int ns = namespace(s); // localize namespace names if (ns != MAIN_NAMESPACE) { - int colon = s.indexOf(':'); + int colon = s.indexOf(":"); s = namespaceIdentifier(ns) + s.substring(colon); } char[] temp = s.toCharArray(); @@ -8600,7 +7583,7 @@ public String normalize(String s) { switch (temp[i]) { - // illegal characters + // illegal characters case '{': case '}': case '<': @@ -8609,10 +7592,13 @@ public String normalize(String s) case ']': case '|': throw new IllegalArgumentException(s + " is an illegal title"); + case '_': + temp[i] = ' '; + break; } } // https://mediawiki.org/wiki/Unicode_normalization_considerations - String temp2 = new String(temp).replaceAll("\\s+", " "); + String temp2 = new String(temp).trim().replaceAll("\\s+", " "); return Normalizer.normalize(temp2, Normalizer.Form.NFC); } @@ -8621,7 +7607,7 @@ public String normalize(String s) * and other write actions. * @since 0.30 */ - protected synchronized void throttle() + private synchronized void throttle() { try { @@ -8641,7 +7627,7 @@ protected synchronized void throttle() * Checks whether the currently logged on user has sufficient rights to * edit/move a protected page. * - * @param pageinfo the output from {@link #getPageInfo} for an article + * @param pageinfo the output from {@link #getPageInfo(java.lang.String)} * @param action what we are doing * @return whether the user can perform the specified action * @throws IOException if a network error occurs @@ -8658,11 +7644,57 @@ protected boolean checkRights(Map pageinfo, String action) throw if (level.equals(FULL_PROTECTION)) return user.isAllowedTo("editprotected"); } - if (Boolean.TRUE.equals(protectionstate.get("cascade"))) + if ((Boolean)protectionstate.get("cascade") == Boolean.TRUE) // can be null return user.isAllowedTo("editprotected"); return true; } + // cookie methods + + /** + * Sets cookies to an unconnected URLConnection and enables gzip + * compression of returned text. + * @param u an unconnected URLConnection + */ + protected void setCookies(URLConnection u) + { + StringBuilder cookie = new StringBuilder(100); + for (Map.Entry entry : cookies.entrySet()) + { + cookie.append(entry.getKey()); + cookie.append("="); + cookie.append(entry.getValue()); + cookie.append("; "); + } + u.setRequestProperty("Cookie", cookie.toString()); + + // enable gzip compression + if (zipped) + u.setRequestProperty("Accept-encoding", "gzip"); + u.setRequestProperty("User-Agent", useragent); + } + + /** + * Grabs cookies from the URL connection provided. + * @param u an unconnected URLConnection + */ + private void grabCookies(URLConnection u) + { + String headerName; + for (int i = 1; (headerName = u.getHeaderFieldKey(i)) != null; i++) + if (headerName.equalsIgnoreCase("Set-Cookie")) + { + String cookie = u.getHeaderField(i); + cookie = cookie.substring(0, cookie.indexOf(';')); + String name = cookie.substring(0, cookie.indexOf('=')); + String value = cookie.substring(cookie.indexOf('=') + 1, cookie.length()); + // these cookies were pruned, but are still sent for some reason? + // TODO: when these cookies are no longer sent, remove this test + if (!value.equals("deleted")) + cookies.put(name, value); + } + } + // logging methods /** @@ -8687,4 +7719,30 @@ protected void logurl(String url, String method) { logger.logp(Level.INFO, "Wiki", method, "Fetching URL {0}", url); } + + // serialization + + /** + * Writes this wiki to a file. + * @param out an ObjectOutputStream to write to + * @throws IOException if there are local IO problems + * @since 0.10 + */ + private void writeObject(ObjectOutputStream out) throws IOException + { + out.defaultWriteObject(); + } + + /** + * Reads a copy of a wiki from a file. + * @param in an ObjectInputStream to read from + * @throws IOException if there are local IO problems + * @throws ClassNotFoundException if we can't recognize the input + * @since 0.10 + */ + private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException + { + in.defaultReadObject(); + statuscounter = statusinterval; // force a status check on next edit + } } diff --git a/src/pattypan/Main.java b/src/pattypan/Main.java index 4724a19..f44ce93 100644 --- a/src/pattypan/Main.java +++ b/src/pattypan/Main.java @@ -84,7 +84,7 @@ public static void main(String[] args) { new String[]{os, Settings.VERSION} ); - Session.WIKI = Wiki.newSession(wiki, scriptPath, protocol); + Session.WIKI = new Wiki(wiki, scriptPath, protocol); launch(args); } } diff --git a/src/pattypan/Session.java b/src/pattypan/Session.java index 073d65e..6ae27a4 100644 --- a/src/pattypan/Session.java +++ b/src/pattypan/Session.java @@ -51,7 +51,7 @@ public final class Session { public static String WIKICODE = ""; public static ArrayList VARIABLES = new ArrayList<>(Arrays.asList("path", "name")); - public static Wiki WIKI = Wiki.newSession("commons.wikimedia.org"); + public static Wiki WIKI = new Wiki("commons.wikimedia.org"); public static ArrayList FILES_TO_UPLOAD = new ArrayList<>(); static { From efe00b7caed625b8d0ae7e580dcecbde02b303a6 Mon Sep 17 00:00:00 2001 From: Abbe98 Date: Fri, 15 Oct 2021 15:16:36 +0200 Subject: [PATCH 04/18] replace replace deprecated javafx FontLoader.computeStringWidth not present in OpenJFX --- src/pattypan/elements/WikiLabel.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/pattypan/elements/WikiLabel.java b/src/pattypan/elements/WikiLabel.java index b4ca816..d3f914e 100644 --- a/src/pattypan/elements/WikiLabel.java +++ b/src/pattypan/elements/WikiLabel.java @@ -23,10 +23,9 @@ */ package pattypan.elements; -import com.sun.javafx.tk.FontLoader; -import com.sun.javafx.tk.Toolkit; import javafx.geometry.Pos; import javafx.scene.control.Label; +import javafx.scene.text.Text; import javafx.scene.text.TextAlignment; import pattypan.Util; @@ -63,8 +62,9 @@ public WikiLabel setClass(String cssClass) { } public WikiLabel setTranslateByHalf(boolean right) { - FontLoader fontLoader = Toolkit.getToolkit().getFontLoader(); - double textWidth = fontLoader.computeStringWidth(this.getText(), this.getFont()); + Text text = new Text(this.getText()); + text.setFont(this.getFont()); + double textWidth = text.getBoundsInLocal().getWidth(); this.setTranslateX(textWidth * 0.5 * (right ? 1 : -1)); return this; } From 3c96b7def73a5ee9b6f22395823774c54d831633 Mon Sep 17 00:00:00 2001 From: Abbe98 Date: Fri, 15 Oct 2021 16:14:54 +0200 Subject: [PATCH 05/18] Update wiki.java side effects: - removed duplication check (additional side effect URL upload works again) - removed unused wmfwiki.java --- src/org/wikipedia/WMFWiki.java | 141 - src/org/wikipedia/Wiki.java | 7990 ++++++++++++++++------------ src/pattypan/Main.java | 2 +- src/pattypan/Session.java | 2 +- src/pattypan/panes/UploadPane.java | 37 +- 5 files changed, 4470 insertions(+), 3702 deletions(-) delete mode 100644 src/org/wikipedia/WMFWiki.java diff --git a/src/org/wikipedia/WMFWiki.java b/src/org/wikipedia/WMFWiki.java deleted file mode 100644 index e611e37..0000000 --- a/src/org/wikipedia/WMFWiki.java +++ /dev/null @@ -1,141 +0,0 @@ -/** - * @(#)WMFWiki.java 0.01 29/03/2011 - * Copyright (C) 2011 - 2015 MER-C and contributors - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version 3 - * of the License, or (at your option) any later version. Additionally - * this file is subject to the "Classpath" exception. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software Foundation, - * Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. - */ - -package org.wikipedia; - -import java.io.*; -import java.net.URLEncoder; -import java.util.*; -import java.util.logging.*; - -/** - * Stuff specific to Wikimedia wikis. - * @author MER-C - * @version 0.01 - */ -public class WMFWiki extends Wiki -{ - /** - * Creates a new WMF wiki that represents the English Wikipedia. - * @deprecated use WMFWiki#createInstance instead - */ - @Deprecated - public WMFWiki() - { - super("en.wikipedia.org"); - } - - /** - * Creates a new WMF wiki that has the given domain name. - * @param domain a WMF wiki domain name e.g. en.wikipedia.org - * @deprecated use WMFWiki#createInstance instead; this will be made private - */ - @Deprecated - public WMFWiki(String domain) - { - super(domain); - } - - /** - * Creates a new WMF wiki that has the given domain name. - * @param domain a WMF wiki domain name e.g. en.wikipedia.org - * @return the constructed Wiki object - */ - public static WMFWiki createInstance(String domain) - { - WMFWiki wiki = new WMFWiki(domain); - wiki.initVars(); - return wiki; - } - - /** - * Returns the list of publicly readable and editable wikis operated by the - * Wikimedia Foundation. - * @return (see above) - * @throws IOException if a network error occurs - */ - public static WMFWiki[] getSiteMatrix() throws IOException - { - WMFWiki wiki = createInstance("en.wikipedia.org"); - wiki.setMaxLag(0); - String line = wiki.fetch("https://en.wikipedia.org/w/api.php?format=xml&action=sitematrix", "WMFWiki.getSiteMatrix"); - ArrayList wikis = new ArrayList(1000); - - // form: - // - for (int x = line.indexOf("url=\""); x >= 0; x = line.indexOf("url=\"", x)) - { - int a = line.indexOf("https://", x) + 8; - int b = line.indexOf('\"', a); - int c = line.indexOf("/>", b); - x = c; - - // check for closed/fishbowl/private wikis - String temp = line.substring(b, c); - if (temp.contains("closed=\"\"") || temp.contains("private=\"\"") || temp.contains("fishbowl=\"\"")) - continue; - wikis.add(createInstance(line.substring(a, b))); - } - int size = wikis.size(); - Logger temp = Logger.getLogger("wiki"); - temp.log(Level.INFO, "WMFWiki.getSiteMatrix", "Successfully retrieved site matrix (" + size + " + wikis)."); - return wikis.toArray(new WMFWiki[size]); - } - - /** - * Get the global usage for a file (requires extension GlobalUsage). - * - * @param title the title of the page (must contain "File:") - * @return the global usage of the file, including the wiki and page the file is used on - * @throws IOException if a network error occurs - * @throws UnsupportedOperationException if namespace(title) != FILE_NAMESPACE - */ - public String[][] getGlobalUsage(String title) throws IOException - { - title = normalize(title); - if (namespace(title) != FILE_NAMESPACE) - throw new UnsupportedOperationException("Cannot retrieve Globalusage for pages other than File pages!"); - String url = query + "prop=globalusage&gulimit=max&rawcontinue=1&titles=" + URLEncoder.encode(title, "UTF-8"); - String next = ""; - ArrayList usage = new ArrayList<>(500); - - do - { - if (!next.isEmpty()) - next = "&gucontinue=" + URLEncoder.encode(next, "UTF-8"); - String line = fetch(url+next, "getGlobalUsageCount"); - - // parse gucontinue if it is there - if (line.contains("")) - next = parseAttribute(line, "gucontinue", 0); - else - next = null; - - for (int i = line.indexOf(" 0; i = line.indexOf("MediaWiki API for most * operations. It is recommended that the server runs the latest version - * of MediaWiki (1.31), otherwise some functions may not work. + * of MediaWiki (1.31), otherwise some functions may not work. This framework + * requires no dependencies outside the core JDK and does not implement any + * functionality added by MediaWiki extensions. *

* Extended documentation is available * here. * All wikilinks are relative to the English Wikipedia and all timestamps are in * your wiki's time zone. - *

+ *

* Please file bug reports here - * or at the Github issue - * tracker. + * or at the Github issue + * tracker. + * + *

Configuration variables

+ *

+ * Some configuration is available through java.util.Properties. + * Set the system property wiki-java.properties to a file path + * where a configuration file is located. The available variables are: + *

    + *
  • maxretries: (default 2) the number of attempts to retry a network + * request before stopping + *
  • connecttimeout: (default 30000) maximum allowed time for a HTTP(s) + * connection to be established in milliseconds + *
  • readtimeout: (default 180000) maximum allowed time for the read + * to take place in milliseconds (needs to be longer, some connections are + * slow and the data volume is large!). + *
  • loguploadsize: (default 22, equivalent to 2^22 = 4 MB) controls + * the log2(size) of each chunk in chunked uploads. Disable chunked uploads + * by setting a large value here (50, equivalent to 2^50 = 1 PB will do). + * Stuff you actually upload must be no larger than 4 GB. + * .
* * @author MER-C and contributors - * @version 0.33 + * @version 0.36 */ -public class Wiki implements Serializable +public class Wiki implements Comparable { - // Master TODO list: - // *Admin stuff - // *More multiqueries - // *Generators (hard) - // NAMESPACES /** @@ -264,6 +284,12 @@ public class Wiki implements Serializable */ public static final String PATROL_LOG = "patrol"; + /** + * Denotes the page creation log. + * @since 0.36 + */ + public static final String PAGE_CREATION_LOG = "create"; + // PROTECTION LEVELS /** @@ -273,14 +299,14 @@ public class Wiki implements Serializable public static final String NO_PROTECTION = "all"; /** - * Denotes semi-protection (i.e. only autoconfirmed users can perform a - * particular action). + * Denotes semi-protection (only autoconfirmed users can perform a + * action). * @since 0.09 */ public static final String SEMI_PROTECTION = "autoconfirmed"; /** - * Denotes full protection (i.e. only admins can perfom a particular action). + * Denotes full protection (only admins can perfom a particular action). * @since 0.09 */ public static final String FULL_PROTECTION = "sysop"; @@ -288,21 +314,21 @@ public class Wiki implements Serializable // ASSERTION MODES /** - * Use no assertions (i.e. 0). + * Use no assertions. * @see #setAssertionMode * @since 0.11 */ public static final int ASSERT_NONE = 0; /** - * Assert that we are logged in (i.e. 1). This is checked every action. + * Assert that we are logged in. This is checked every action. * @see #setAssertionMode * @since 0.30 */ public static final int ASSERT_USER = 1; /** - * Assert that we have a bot flag (i.e. 2). This is checked every action. + * Assert that we have a bot flag. This is checked every action. * @see #setAssertionMode * @since 0.11 */ @@ -317,74 +343,41 @@ public class Wiki implements Serializable public static final int ASSERT_NO_MESSAGES = 4; /** - * Assert that we have a sysop flag (i.e. 8). This is checked intermittently. + * Assert that we have a sysop flag. This is checked intermittently. * @see #setAssertionMode * @since 0.30 */ public static final int ASSERT_SYSOP = 8; - // RC OPTIONS - - /** - * In queries against the recent changes table, this would mean we don't - * fetch anonymous edits. - * @since 0.20 - */ - public static final int HIDE_ANON = 1; - - /** - * In queries against the recent changes table, this would mean we don't - * fetch edits made by bots. - * @since 0.20 - */ - public static final int HIDE_BOT = 2; - - /** - * In queries against the recent changes table, this would mean we don't - * fetch by the logged in user. - * @since 0.20 - */ - public static final int HIDE_SELF = 4; - - /** - * In queries against the recent changes table, this would mean we don't - * fetch minor edits. - * @since 0.20 - */ - public static final int HIDE_MINOR = 8; - - /** - * In queries against the recent changes table, this would mean we don't - * fetch patrolled edits. - * @since 0.20 - */ - public static final int HIDE_PATROLLED = 16; - // REVISION OPTIONS /** - * In Revision.diff(), denotes the next revision. - * @see org.wikipedia.Wiki.Revision#diff(org.wikipedia.Wiki.Revision) + * In {@link org.wikipedia.Wiki.Revision#diff(long) Revision.diff()}, + * denotes the next revision. + * @see org.wikipedia.Wiki.Revision#diff(long) * @since 0.21 */ public static final long NEXT_REVISION = -1L; /** - * In Revision.diff(), denotes the current revision. - * @see org.wikipedia.Wiki.Revision#diff(org.wikipedia.Wiki.Revision) + * In {@link org.wikipedia.Wiki.Revision#diff(long) Revision.diff()}, + * denotes the current revision. + * @see org.wikipedia.Wiki.Revision#diff(long) * @since 0.21 */ public static final long CURRENT_REVISION = -2L; /** - * In Revision.diff(), denotes the previous revision. - * @see org.wikipedia.Wiki.Revision#diff(org.wikipedia.Wiki.Revision) + * In {@link org.wikipedia.Wiki.Revision#diff(long) Revision.diff()}, + * denotes the previous revision. + * @see org.wikipedia.Wiki.Revision#diff(long) * @since 0.21 */ public static final long PREVIOUS_REVISION = -3L; /** * The list of options the user can specify for his/her gender. + * @see User#getGender() * @since 0.24 */ public enum Gender @@ -412,230 +405,290 @@ public enum Gender unknown; } - private static final String version = "0.33"; + private static final String version = "0.36"; + + // fundamental URL strings + private final String protocol, domain, scriptPath; + private String base, articleUrl; + + /** + * Stores default HTTP parameters for API calls. Contains {@linkplain + * #setMaxLag(int) maxlag}, {@linkplain #setResolveRedirects(boolean) redirect + * resolution} and {@linkplain #setAssertionMode(int) user and bot assertions} + * when wanted by default. Add stuff to this map if you want to add parameters + * to every API call. + * @see #makeApiCall(Map, Map, String) + */ + protected ConcurrentHashMap defaultApiParams; + + /** + * URL entrypoint for the MediaWiki API. (Needs to be accessible to + * subclasses.) + * @see #initVars() + * @see #getApiUrl() + * @see MediaWiki + * documentation + */ + protected String apiUrl; - // the domain of the wiki - private String domain; - protected String query, base, apiUrl; - protected String scriptPath = "/w"; + // wiki properties + private boolean siteinfofetched = false; private boolean wgCapitalLinks = true; - private ZoneId timezone = ZoneId.of("UTC"); + private String dbname; + private String mwVersion; + private ZoneId timezone = ZoneOffset.UTC; + private Locale locale = Locale.ENGLISH; + private List extensions = Collections.emptyList(); + private LinkedHashMap namespaces = null; + private ArrayList ns_subpages = null; // user management - private Map cookies = new HashMap<>(12); + private HttpClient client; + private final CookieManager cookies; private User user; private int statuscounter = 0; - // various caches - private transient LinkedHashMap namespaces = null; - private transient ArrayList ns_subpages = null; - private transient List watchlist = null; + // watchlist cache + private List watchlist = null; // preferences private int max = 500; private int slowmax = 50; - private int throttle = 10000; // throttle + private int throttle = 10000; private int maxlag = 5; private int assertion = ASSERT_NONE; // assertion mode - private transient int statusinterval = 100; // status check + private int statusinterval = 100; // status check private int querylimit = Integer.MAX_VALUE; private String useragent = "Wiki.java/" + version + " (https://github.com/MER-C/wiki-java/)"; private boolean zipped = true; private boolean markminor = false, markbot = false; private boolean resolveredirect = false; - private String protocol = "https://"; private Level loglevel = Level.ALL; private static final Logger logger = Logger.getLogger("wiki"); // Store time when the last throttled action was executed private long lastThrottleActionTime = 0; - // retry count - private int maxtries = 2; - - // serial version - private static final long serialVersionUID = -8745212681497643456L; - - // time to open a connection - private static final int CONNECTION_CONNECT_TIMEOUT_MSEC = 30000; // 30 seconds - // time for the read to take place. (needs to be longer, some connections are slow - // and the data volume is large!) - private static final int CONNECTION_READ_TIMEOUT_MSEC = 180000; // 180 seconds - // log2(upload chunk size). Default = 22 => upload size = 4 MB. Disable - // chunked uploads by setting a large value here (50 = 1 PB will do). - private static final int LOG2_CHUNK_SIZE = 22; - // maximum URL length in bytes - protected static final int URL_LENGTH_LIMIT = 8192; + // config via properties + private final int maxtries; + private final int read_timeout_msec; + private final int log2_upload_size; // CONSTRUCTORS AND CONFIGURATION /** - * Creates a new connection to the English Wikipedia via HTTPS. - * @since 0.02 - * @deprecated use Wiki#createInstance instead - */ - @Deprecated - public Wiki() - { - this("en.wikipedia.org", "/w"); - } - - /** - * Creates a new connection to a wiki via HTTPS. WARNING: if the wiki uses - * a $wgScriptpath other than the default /w, you need to call - * getScriptPath() to automatically set it. Alternatively, you - * can use the constructor below if you know it in advance. - * - * @param domain the wiki domain name e.g. en.wikipedia.org (defaults to - * en.wikipedia.org) - * @deprecated use Wiki#createInstance instead - */ - @Deprecated - public Wiki(String domain) - { - this(domain, "/w"); - } - - /** - * Creates a new connection to a wiki with $wgScriptpath set to - * scriptPath via HTTPS. - * - * @param domain the wiki domain name - * @param scriptPath the script path - * @since 0.14 - * @deprecated use Wiki#createInstance instead - */ - @Deprecated - public Wiki(String domain, String scriptPath) - { - this(domain, scriptPath, "https://"); - } - - /** - * Creates a new connection to a wiki with $wgScriptpath set to - * scriptPath via the specified protocol. + * Creates a new MediaWiki API client for the given wiki with + * $wgScriptPath set to scriptPath and via the + * specified protocol. * * @param domain the wiki domain name * @param scriptPath the script path * @param protocol a protocol e.g. "http://", "https://" or "file:///" * @since 0.31 - * @deprecated use Wiki#createInstance instead; this will be made private - * for the reason in the TODO comment below */ - @Deprecated - public Wiki(String domain, String scriptPath, String protocol) - { - if (domain == null || domain.isEmpty()) - domain = "en.wikipedia.org"; - this.domain = domain; - this.scriptPath = scriptPath; - this.protocol = protocol; - - // init variables - // This is fine as long as you do not have parameters other than domain - // and scriptpath in constructors and do not do anything else than super(x)! - // http://stackoverflow.com/questions/3404301/whats-wrong-with-overridable-method-calls-in-constructors - // TODO: remove this + protected Wiki(String domain, String scriptPath, String protocol) + { + this.domain = Objects.requireNonNull(domain); + this.scriptPath = Objects.requireNonNull(scriptPath); + this.protocol = Objects.requireNonNull(protocol); + + defaultApiParams = new ConcurrentHashMap<>(); + defaultApiParams.put("format", "xml"); + defaultApiParams.put("maxlag", String.valueOf(maxlag)); + logger.setLevel(loglevel); logger.log(Level.CONFIG, "[{0}] Using Wiki.java {1}", new Object[] { domain, version }); - initVars(); + + // read in config + Properties props = new Properties(); + String filename = System.getProperty("wiki-java.properties"); + if (filename != null) + { + try + { + InputStream in = new FileInputStream(new File(filename)); + props.load(in); + } + catch (IOException ex) + { + logger.log(Level.WARNING, "Unable to load properties file " + filename); + } + } + maxtries = Integer.parseInt(props.getProperty("maxretries", "2")); + log2_upload_size = Integer.parseInt(props.getProperty("loguploadsize", "22")); // 4 MB + read_timeout_msec = Integer.parseInt(props.getProperty("readtimeout", "180000")); // 180 seconds + cookies = new CookieManager(null, CookiePolicy.ACCEPT_ALL); + client = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(30)) + .cookieHandler(cookies) + .build(); } - + /** - * Creates a new connection to a wiki via HTTPS. Depending on the settings - * of the wiki, you may need to call {@link Wiki#getSiteInfo()} on the - * returned object after this in order for some functionality to work - * correctly. + * Creates a new MediaWiki API client for the given wiki using HTTPS. + * Depending on the settings of the wiki, you may need to call {@link + * Wiki#getSiteInfo()} on the returned object after this in order for some + * functionality to work correctly. * * @param domain the wiki domain name e.g. en.wikipedia.org (defaults to * en.wikipedia.org) - * @return the created wiki + * @return the constructed API client object * @since 0.34 */ - public static Wiki createInstance(String domain) + public static Wiki newSession(String domain) { - return createInstance(domain, "/w", "https://"); + return newSession(domain, "/w", "https://"); } - + /** - * Creates a new connection to a wiki with - * $wgScriptPath set to scriptPath and via the - * specified protocol. Depending on the settings of the wiki, you may need - * to call {@link Wiki#getSiteInfo()} on the returned object after this in + * $wgScriptPath set to scriptPath and via the + * specified protocol. Depending on the settings of the wiki, you may need + * to call {@link Wiki#getSiteInfo()} on the returned object after this in * order for some functionality to work correctly. * + *

All factory methods in subclasses must call {@link #initVars()}. + * * @param domain the wiki domain name * @param scriptPath the script path * @param protocol a protocol e.g. "http://", "https://" or "file:///" - * @return the constructed Wiki object + * @return the constructed API client object * @since 0.34 */ - public static Wiki createInstance(String domain, String scriptPath, String protocol) + public static Wiki newSession(String domain, String scriptPath, String protocol) { // Don't put network requests here. Servlets cannot afford to make // unnecessary network requests in initialization. - Wiki wiki = new Wiki(domain, "/w", protocol); - wiki.initVars(); // construct URL bases + Wiki wiki = new Wiki(domain, scriptPath, protocol); + wiki.initVars(); return wiki; } /** * Edit this if you need to change the API and human interface url - * configuration of the wiki. One example use is server-side cache - * management (maxage and smaxage API parameters). + * configuration of the wiki. One example use is to change the port number. * *

Contributed by Tedder * @since 0.24 */ protected void initVars() { - StringBuilder basegen = new StringBuilder(protocol); - basegen.append(domain); - basegen.append(scriptPath); - StringBuilder apigen = new StringBuilder(basegen); - apigen.append("/api.php?format=xml&"); - // MediaWiki has inbuilt maxlag functionality, see [[mw:Manual:Maxlag - // parameter]]. Let's exploit it. - if (maxlag >= 0) - { - apigen.append("maxlag="); - apigen.append(maxlag); - apigen.append("&"); - basegen.append("/index.php?maxlag="); - basegen.append(maxlag); - basegen.append("&title="); - } - else - basegen.append("/index.php?title="); - base = basegen.toString(); - // the native API supports assertions as of MW 1.23 - if ((assertion & ASSERT_BOT) == ASSERT_BOT) - apigen.append("assert=bot&"); - else if ((assertion & ASSERT_USER) == ASSERT_USER) - apigen.append("assert=user&"); - apiUrl = apigen.toString(); - apigen.append("action=query&"); - if (resolveredirect) - apigen.append("redirects&"); - query = apigen.toString(); + base = protocol + domain + scriptPath + "/index.php"; + apiUrl = protocol + domain + scriptPath + "/api.php"; + articleUrl = protocol + domain + "/wiki/"; } /** - * Gets the domain of the wiki, as supplied on construction. + * Gets the domain of the wiki as supplied on construction. * @return the domain of the wiki * @since 0.06 */ - public String getDomain() + public final String getDomain() { return domain; } - - public String getProtocol() + + /** + * Gets the + * $wgScriptPath variable as supplied on construction. + * @return the script path of the wiki + * @since 0.14 + */ + public final String getScriptPath() + { + return scriptPath; + } + + /** + * Gets the protocol used to access this MediaWiki instance, as supplied + * on construction. + * @return (see above) + * @since 0.35 + */ + public final String getProtocol() { return protocol; } + /** + * Determines whether this wiki is equal to another object based on the + * protocol (not case sensitive), domain (not case sensitive) and + * scriptPath (case sensitive). A return value of {@code true} means two + * Wiki objects point to the same instance of MediaWiki. + * @param obj the object to compare + * @return whether the wikis point to the same instance of MediaWiki + */ + @Override + public boolean equals(Object obj) + { + if (!(obj instanceof Wiki)) + return false; + Wiki other = (Wiki)obj; + return domain.equalsIgnoreCase(other.domain) + && scriptPath.equals(other.scriptPath) + && protocol.equalsIgnoreCase(other.protocol); + } + + /** + * Returns a hash code of this object based on the protocol, domain and + * scriptpath. + * @return a hash code + */ + @Override + public int hashCode() + { + // English locale used here for reproducability and so network requests + // are not required + int hc = domain.toLowerCase(Locale.ENGLISH).hashCode(); + hc = 127 * hc + scriptPath.hashCode(); + hc = 127 * hc + protocol.toLowerCase(Locale.ENGLISH).hashCode(); + return hc; + } + + /** + * Allows wikis to be sorted based on their domain (case insensitive), then + * their script path (case sensitive). If 0 is returned, it is reasonable + * both Wikis point to the same instance of MediaWiki. + * @param other the wiki to compare to + * @return -1 if this wiki is alphabetically before the other, 1 if after + * and 0 if they are likely to be the same instance of MediaWiki + * @since 0.35 + */ + @Override + public int compareTo(Wiki other) + { + int result = domain.compareToIgnoreCase(other.domain); + if (result == 0) + result = scriptPath.compareTo(other.scriptPath); + return result; + } + + /** + * Gets the URL of index.php. + * @return (see above) + * @see + * MediaWiki documentation + * @since 0.35 + */ + public String getIndexPhpUrl() + { + return base; + } + + /** + * Gets the URL of api.php. + * @return (see above) + * @see MediaWiki + * documentation + * @since 0.36 + */ + public String getApiUrl() + { + return apiUrl; + } + /** * Gets the editing throttle. * @return the throttle value in milliseconds @@ -648,8 +701,11 @@ public int getThrottle() } /** - * Sets the editing throttle. Read requests are not throttled or restricted - * in any way. Default is 10s. + * Sets the throttle, which limits most write requests to no more than one + * per wiki instance in the given time across all threads. (As a + * consequence, all throttled methods are thread safe.) Read requests are + * not throttled or restricted in any way. Default is 10 seconds. + * * @param throttle the new throttle value in milliseconds * @see #getThrottle * @since 0.09 @@ -662,69 +718,166 @@ public void setThrottle(int throttle) /** * Gets various properties of the wiki and sets the bot framework up to use - * them. Also populates the namespace cache. Returns: + * them. The return value is cached. This method is thread safe. Returns: *

    *
  • usingcapitallinks: (Boolean) whether a wiki forces upper case * for the title. Example: en.wikipedia = true, en.wiktionary = false. * Default = true. See * $wgCapitalLinks - *
  • scriptpath: (String) the scriptpath: (String) the - * $wgScriptPath wiki variable. Default = /w. + * $wgScriptPath wiki variable. Default = {@code /w}. *
  • version: (String) the MediaWiki version used for this wiki - *
  • timezone: (String) the timezone the wiki is in, default = UTC + *
  • timezone: (ZoneId) the timezone the wiki is in, default = UTC + *
  • locale: (Locale) the locale of the wiki + *
  • dbname: (String) the internal name of the database *
* * @return (see above) * @since 0.30 * @throws IOException if a network error occurs + * @deprecated This method is likely going to get renamed with the return + * type changed to void once I finish cleaning up the site info caching + * mechanism. Use the specialized methods instead. */ - public Map getSiteInfo() throws IOException - { - Map ret = new HashMap<>(); - String line = fetch(query + "action=query&meta=siteinfo&siprop=namespaces%7Cnamespacealiases%7Cgeneral", "getSiteInfo"); - - // general site info - String bits = line.substring(line.indexOf("")); - wgCapitalLinks = parseAttribute(bits, "case", 0).equals("first-letter"); - ret.put("usingcapitallinks", wgCapitalLinks); - scriptPath = parseAttribute(bits, "scriptpath", 0); - ret.put("scriptpath", scriptPath); - timezone = ZoneId.of(parseAttribute(bits, "timezone", 0)); - ret.put("timezone", timezone); - ret.put("version", parseAttribute(bits, "generator", 0)); - - // populate namespace cache - namespaces = new LinkedHashMap<>(30); - ns_subpages = new ArrayList<>(30); - // xml form: Media or - String[] items = line.split("') + 1; - int c = items[i].indexOf(""); - if (c < 0) - namespaces.put("", ns); - else - namespaces.put(normalize(decode(items[i].substring(b, c))), ns); - - String canonicalnamespace = parseAttribute(items[i], "canonical", 0); - if (canonicalnamespace != null) - namespaces.put(canonicalnamespace, ns); - - // does this namespace support subpages? - if (items[i].contains("subpages=\"\"")) - ns_subpages.add(ns); + @Deprecated + public synchronized Map getSiteInfo() throws IOException + { + Map siteinfo = new HashMap<>(); + if (!siteinfofetched) + { + Map getparams = new HashMap<>(); + getparams.put("action", "query"); + getparams.put("meta", "siteinfo"); + getparams.put("siprop", "namespaces|namespacealiases|general|extensions"); + String line = makeApiCall(getparams, null, "getSiteInfo"); + detectUncheckedErrors(line, null, null); + + // general site info + String bits = line.substring(line.indexOf("")); + wgCapitalLinks = parseAttribute(bits, "case", 0).equals("first-letter"); + timezone = ZoneId.of(parseAttribute(bits, "timezone", 0)); + mwVersion = parseAttribute(bits, "generator", 0); + locale = new Locale(parseAttribute(bits, "lang", 0)); + dbname = parseAttribute(bits, "wikiid", 0); + + // parse extensions + bits = line.substring(line.indexOf(""), line.indexOf("")); + extensions = new ArrayList<>(); + String[] unparsed = bits.split("(30); + ns_subpages = new ArrayList<>(30); + // xml form: Media or + String[] items = line.split("') + 1; + int c = items[i].indexOf(""); + if (c < 0) + namespaces.put("", ns); + else + namespaces.put(normalize(decode(items[i].substring(b, c))), ns); + + String canonicalnamespace = parseAttribute(items[i], "canonical", 0); + if (canonicalnamespace != null) + namespaces.put(canonicalnamespace, ns); + + // does this namespace support subpages? + if (items[i].contains("subpages=\"\"")) + ns_subpages.add(ns); + } + siteinfofetched = true; + log(Level.INFO, "getSiteInfo", "Successfully retrieved site info for " + getDomain()); } - - initVars(); - log(Level.INFO, "getSiteInfo", "Successfully retrieved site info for " + getDomain()); - return ret; + siteinfo.put("usingcapitallinks", wgCapitalLinks); + siteinfo.put("scriptpath", scriptPath); + siteinfo.put("timezone", timezone); + siteinfo.put("version", mwVersion); + siteinfo.put("locale", locale); + siteinfo.put("extensions", extensions); + siteinfo.put("dbname", dbname); + return siteinfo; + } + + /** + * Gets the version of MediaWiki this wiki runs e.g. 1.20wmf5 (54b4fcb). + * See [[Special:Version]] on your wiki. + * @return (see above) + * @throws UncheckedIOException if the site info cache has not been + * populated and a network error occurred when populating it + * @since 0.14 + * @see MediaWiki Git + */ + public String version() + { + ensureNamespaceCache(); + return mwVersion; + } + + /** + * Detects whether a wiki forces upper case for the first character in a + * title. Example: en.wikipedia = true, en.wiktionary = false. + * @return (see above) + * @throws UncheckedIOException if the site info cache has not been + * populated and a network error occurred when populating it + * @see MediaWiki + * documentation + * @since 0.30 + */ + public boolean usesCapitalLinks() + { + ensureNamespaceCache(); + return wgCapitalLinks; + } + + /** + * Returns the list of extensions installed on this wiki. + * @return (see above) + * @throws UncheckedIOException if the site info cache has not been + * populated and a network error occurred when populating it + * @see MediaWiki + * documentation + * @since 0.35 + */ + public List installedExtensions() + { + ensureNamespaceCache(); + return new ArrayList<>(extensions); + } + + /** + * Gets the timezone of this wiki + * @return (see above) + * @throws UncheckedIOException if the site info cache has not been + * populated and a network error occurred when populating it + * @since 0.35 + */ + public ZoneId timezone() + { + ensureNamespaceCache(); + return timezone; + } + + /** + * Gets the locale of this wiki. + * @return (see above) + * @throws UncheckedIOException if the site info cache has not been + * populated and a network error occurred when populating it + * @since 0.35 + */ + public Locale locale() + { + ensureNamespaceCache(); + return locale; } /** @@ -753,7 +906,9 @@ public String getUserAgent() * Enables/disables GZip compression for GET requests. Default: true. * @param zipped whether we use GZip compression * @since 0.23 + * @deprecated this is now handled transparently; just delete calls to this method. */ + @Deprecated(forRemoval=true) public void setUsingCompressedRequests(boolean zipped) { this.zipped = zipped; @@ -764,7 +919,9 @@ public void setUsingCompressedRequests(boolean zipped) * Default: true. * @return (see above) * @since 0.23 + * @deprecated this is now handled transparently; just delete calls to this method. */ + @Deprecated(forRemoval=true) public boolean isUsingCompressedRequests() { return zipped; @@ -790,7 +947,10 @@ public boolean isResolvingRedirects() public void setResolveRedirects(boolean b) { resolveredirect = b; - initVars(); + if (b) + defaultApiParams.put("redirects", "1"); + else + defaultApiParams.remove("redirects"); } /** @@ -863,31 +1023,6 @@ public void setQueryLimit(int limit) querylimit = limit; } - /** - * Determines whether this wiki is equal to another object. - * @param obj the object to compare - * @return whether the domains of the wikis are equal - * @since 0.10 - */ - @Override - public boolean equals(Object obj) - { - if (!(obj instanceof Wiki)) - return false; - return domain.equals(((Wiki)obj).domain); - } - - /** - * Returns a hash code of this object. - * @return a hash code - * @since 0.12 - */ - @Override - public int hashCode() - { - return domain.hashCode() * maxlag - throttle; - } - /** * Returns a string representation of this Wiki. * @return a string representation of this Wiki. @@ -897,12 +1032,14 @@ public int hashCode() public String toString() { // domain - StringBuilder buffer = new StringBuilder("Wiki[domain="); + StringBuilder buffer = new StringBuilder("Wiki[url="); + buffer.append(protocol); buffer.append(domain); + buffer.append(scriptPath); // user buffer.append(",user="); - buffer.append(user != null ? user.toString() : "null"); + buffer.append(Objects.toString(user)); buffer.append(","); // throttle mechanisms @@ -948,7 +1085,10 @@ public void setMaxLag(int lag) { maxlag = lag; log(Level.CONFIG, "setMaxLag", "Setting maximum allowable database lag to " + lag); - initVars(); + if (maxlag >= 0) + defaultApiParams.put("maxlag", String.valueOf(maxlag)); + else + defaultApiParams.remove("maxlag"); } /** @@ -973,7 +1113,13 @@ public void setAssertionMode(int mode) { assertion = mode; log(Level.CONFIG, "setAssertionMode", "Set assertion mode to " + mode); - initVars(); + + if ((assertion & ASSERT_BOT) == ASSERT_BOT) + defaultApiParams.put("assert", "bot"); + else if ((assertion & ASSERT_USER) == ASSERT_USER) + defaultApiParams.put("assert", "user"); + else + defaultApiParams.remove("assert"); } /** @@ -1021,29 +1167,36 @@ public void setLogLevel(Level loglevel) // META STUFF /** - * Logs in to the wiki. This method is thread-safe. + * Logs in to the wiki. This method is thread-safe. * * @param username a username - * @param password a password (as a char[] due to JPasswordField) + * @param password a password, as a {@code char[]} for security + * reasons. Overwritten once the password is used. * @throws IOException if a network error occurs + * @throws FailedLoginException if the login failed due to an incorrect + * username or password, the requirement for an interactive login (not + * supported, use [[Special:BotPasswords]]) or some other reason * @see #logout + * @see MediaWiki + * documentation */ public synchronized void login(String username, char[] password) throws IOException, FailedLoginException { - StringBuilder buffer = new StringBuilder(500); - buffer.append("lgname="); - buffer.append(encode(username, false)); - buffer.append("&lgpassword="); - buffer.append(encode(new String(password), false)); - buffer.append("&lgtoken="); - buffer.append(encode(getToken("login"), false)); - String line = post(apiUrl + "action=login", buffer.toString(), "login"); - buffer.setLength(0); + Map getparams = new HashMap<>(); + getparams.put("action", "login"); + Map postparams = new HashMap<>(); + postparams.put("lgname", username); + postparams.put("lgpassword", new String(password)); + postparams.put("lgtoken", getToken("login")); + String line = makeApiCall(getparams, postparams, "login"); + detectUncheckedErrors(line, null, null); + Arrays.fill(password, '0'); // check for success if (line.contains("result=\"Success\"")) { - user = new User(parseAttribute(line, "lgusername", 0)); + String returned_username = parseAttribute(line, "lgusername", 0); + user = getUsers(List.of(returned_username)).get(0); boolean apihighlimit = user.isAllowedTo("apihighlimits"); if (apihighlimit) { @@ -1055,18 +1208,21 @@ public synchronized void login(String username, char[] password) throws IOExcept else if (line.contains("result=\"Failed\"")) throw new FailedLoginException("Login failed: " + parseAttribute(line, "reason", 0)); // interactive login or bot password required - else if (line.contains("result=\"Aborted\"")) + else if (line.contains("result=\"Aborted\"")) throw new FailedLoginException("Login failed: you need to use a bot password, see [[Special:Botpasswords]]."); else throw new AssertionError("Unreachable!"); } /** - * Logs in to the wiki. This method is thread-safe. + * Logs in to the wiki. This method is thread-safe. * * @param username a username * @param password a string with the password * @throws IOException if a network error occurs + * @throws FailedLoginException if the login failed due to an incorrect + * username or password, the requirement for an interactive login (not + * supported, use [[Special:Botpasswords]]) or some other reason * @see #logout */ public synchronized void login(String username, String password) throws IOException, FailedLoginException @@ -1083,7 +1239,7 @@ public synchronized void login(String username, String password) throws IOExcept */ public synchronized void logout() { - cookies.clear(); + cookies.getCookieStore().removeAll(); user = null; max = 500; slowmax = 50; @@ -1102,10 +1258,15 @@ public synchronized void logout() * @since 0.14 * @see #login * @see #logout + * @see MediaWiki + * documentation */ public synchronized void logoutServerSide() throws IOException { - fetch(apiUrl + "action=logout", "logoutServerSide"); + Map getparams = new HashMap<>(); + getparams.put("action", "logout"); + String response = makeApiCall(getparams, null, "logoutServerSide"); + detectUncheckedErrors(response, null, null); logout(); // destroy local cookies } @@ -1118,12 +1279,18 @@ public synchronized void logoutServerSide() throws IOException */ public boolean hasNewMessages() throws IOException { - String url = query + "meta=userinfo&uiprop=hasmsg"; - return fetch(url, "hasNewMessages").contains("messages=\"\""); + Map getparams = new HashMap<>(); + getparams.put("action", "query"); + getparams.put("meta", "userinfo"); + getparams.put("uiprop", "hasmsg"); + String response = makeApiCall(getparams, null, "hasNewMessages"); + detectUncheckedErrors(response, null, null); + return response.contains("messages=\"\""); } /** - * Determines the current database replication lag. + * Determines the current database replication lag. This method does not + * wait if the maxlag setting is exceeded. This method is thread safe. * @return the current database replication lag * @throws IOException if a network error occurs * @see #setMaxLag @@ -1132,12 +1299,26 @@ public boolean hasNewMessages() throws IOException * MediaWiki documentation * @since 0.10 */ - public int getCurrentDatabaseLag() throws IOException + public double getCurrentDatabaseLag() throws IOException { - String line = fetch(query + "meta=siteinfo&siprop=dbrepllag", "getCurrentDatabaseLag"); - String lag = parseAttribute(line, "lag", 0); - log(Level.INFO, "getCurrentDatabaseLag", "Current database replication lag is " + lag + " seconds"); - return Integer.parseInt(lag); + Map getparams = new HashMap<>(); + getparams.put("action", "query"); + getparams.put("meta", "siteinfo"); + getparams.put("siprop", "dbrepllag"); + + synchronized (this) + { + // bypass lag check for this request + int temp = getMaxLag(); + setMaxLag(-1); + String line = makeApiCall(getparams, null, "getCurrentDatabaseLag"); + detectUncheckedErrors(line, null, null); + setMaxLag(temp); + + String lag = parseAttribute(line, "lag", 0); + log(Level.INFO, "getCurrentDatabaseLag", "Current database replication lag is " + lag + " seconds"); + return Double.parseDouble(lag); + } } /** @@ -1152,7 +1333,12 @@ public int getCurrentDatabaseLag() throws IOException */ public Map getSiteStatistics() throws IOException { - String text = fetch(query + "meta=siteinfo&siprop=statistics", "getSiteStatistics"); + Map getparams = new HashMap<>(); + getparams.put("action", "query"); + getparams.put("meta", "siteinfo"); + getparams.put("siprop", "statistics"); + String text = makeApiCall(getparams, null, "getSiteStatistics"); + detectUncheckedErrors(text, null, null); Map ret = new HashMap<>(20); ret.put("pages", Integer.parseInt(parseAttribute(text, "pages", 0))); ret.put("articles", Integer.parseInt(parseAttribute(text, "articles", 0))); @@ -1165,10 +1351,8 @@ public Map getSiteStatistics() throws IOException } /** - * Renders the specified wiki markup by passing it to the MediaWiki - * parser through the API. (Note: this isn't implemented locally because - * I can't be stuffed porting Parser.php). One use of this method is to - * emulate the previewing functionality of the MediaWiki software. + * Renders the specified wiki markup as HTML by passing it to the MediaWiki + * parser through the API. * * @param markup the markup to parse * @return the parsed markup as HTML @@ -1177,32 +1361,99 @@ public Map getSiteStatistics() throws IOException */ public String parse(String markup) throws IOException { - // This is POST because markup can be arbitrarily large, as in the size - // of an article (over 10kb). - String response = post(apiUrl + "action=parse", "prop=text&text=" + encode(markup, false), "parse"); - int y = response.indexOf('>', response.indexOf(""); - return decode(response.substring(y, z)); + Map content = new HashMap<>(); + content.put("text", markup); + return parse(content, -1, false); } /** - * Same as {@link #parse(java.lang.String)}, but also strips out unwanted - * crap. This might be useful to subclasses. + * Parses wikitext, revisions or pages. Deleted pages and revisions to + * deleted pages are not allowed if you don't have the rights to view them. * - * @param in the string to parse - * @return that string without the crap - * @throws IOException if a network error occurs - * @since 0.14 - */ - protected String parseAndCleanup(String in) throws IOException - { - String output = parse(in); - output = output.replace("

", "").replace("

", ""); // remove paragraph tags - output = output.replace("\n", ""); // remove new lines - - // strip out the parser report, which comes at the end - int a = output.indexOf("" nocreate="" allowusertalk=""/> + for (int a = line.indexOf(" 0; a = line.indexOf(" - for (int a = line.indexOf(" 0; a = line.indexOf("", a); - String temp = line.substring(a, b); - LogEntry le = parseLogEntry(temp); - le.type = BLOCK_LOG; - le.action = "block"; - // parseLogEntries parses block target into le.user due to mw.api - // attribute name - if (le.user == null) // autoblock - le.target = "#" + parseAttribute(temp, "id", 0); - else - le.target = namespaceIdentifier(USER_NAMESPACE) + ":" + le.user.username; - // parse blocker for real - le.user = new User(parseAttribute(temp, "by", 0)); - results.add(le); - } + // find entry + int b = Math.max(line.indexOf("/>", a), line.indexOf("", a)); + String temp = line.substring(a, b); + + String blocker = parseAttribute(temp, "by", 0); + String blockeduser = parseAttribute(temp, "user", 0); + String target; + if (blockeduser == null) // autoblock + target = "#" + parseAttribute(temp, "id", 0); + else + target = namespaceIdentifier(USER_NAMESPACE) + ":" + blockeduser; + + LogEntry le = parseLogEntry(temp, blocker, BLOCK_LOG, "block", target); + results.add(le); } - catch (IOException ex) + }; + List entries = new ArrayList<>(); + if (users == null) + entries.addAll(makeListQuery("bk", getparams, null, "getBlockList", limit, parser)); + else + { + for (String bkusers : constructTitleString(users)) { - throw new UncheckedIOException(ex); + getparams.put("bkusers", bkusers); + entries.addAll(makeListQuery("bk", getparams, null, "getBlockList", limit, parser)); } - }); - - // log statement - StringBuilder logRecord = new StringBuilder("Successfully fetched IP block list "); - if (!user.isEmpty()) - { - logRecord.append(" for "); - logRecord.append(user); - } - if (start != null) - { - logRecord.append(" from "); - logRecord.append(start.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - } - if (end != null) - { - logRecord.append(" to "); - logRecord.append(end.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); } - int size = entries.size(); - logRecord.append(" ("); - logRecord.append(size); - logRecord.append(" entries)"); - log(Level.INFO, "getIPBlockList", logRecord.toString()); - return entries.toArray(new LogEntry[size]); - } - - /** - * Gets the log entries representing actions that were performed on a - * specific target. Equivalent to [[Special:Log]]. - * @param logtype what log to get (e.g. {@link #DELETION_LOG}) - * @param action what action to get (e.g. delete, undelete, etc.), use - * null to not specify one - * @param user the user performing the action. Use null not to specify - * one. - * @param target the target of the action(s). - * @throws IOException if a network error occurs - * @return the specified log entries - * @since 0.08 - */ - public LogEntry[] getLogEntries(String logtype, String action, String user, String target) throws IOException - { - return getLogEntries(logtype, action, user, target, null, null, Integer.MAX_VALUE, ALL_NAMESPACES); - } - - /** - * Gets the last how ever many log entries in the specified log. Equivalent - * to [[Special:Log]] and [[Special:Newimages]] when - * type.equals({@link #UPLOAD_LOG}). - * - * @param logtype what log to get (e.g. {@link #DELETION_LOG}) - * @param action what action to get (e.g. delete, undelete, etc.), use - * null to not specify one - * @param amount the number of entries to get (overrides global limits) - * @throws IOException if a network error occurs - * @throws IllegalArgumentException if the log type doesn't exist - * @return the specified log entries - */ - public LogEntry[] getLogEntries(String logtype, String action, int amount) throws IOException - { - return getLogEntries(logtype, action, null, null, null, null, amount, ALL_NAMESPACES); + log(Level.INFO, "getBlockList", "Successfully fetched block list " + entries.size() + " entries)"); + return entries; } /** * Gets the specified amount of log entries between the given times by * the given user on the given target. Equivalent to [[Special:Log]]. - * WARNING: the start date is the most recent of the dates given, and - * the order of enumeration is from newest to oldest. + * Accepted parameters from helper are: + * + *
    + *
  • {@link Wiki.RequestHelper#withinDateRange(OffsetDateTime, + * OffsetDateTime) date range} + *
  • {@link Wiki.RequestHelper#byUser(String) user} + *
  • {@link Wiki.RequestHelper#byTitle(String) title} + *
  • {@link Wiki.RequestHelper#reverse(boolean) reverse} + *
  • {@link Wiki.RequestHelper#inNamespaces(int...) namespaces} (one + * namespace only, must not be used if a title is specified) + *
  • {@link Wiki.RequestHelper#taggedWith(String) tag} + *
  • {@link Wiki.RequestHelper#limitedTo(int) local query limit} + *
* * @param logtype what log to get (e.g. {@link #DELETION_LOG}) - * @param action what action to get (e.g. delete, undelete, etc.), use - * null to not specify one - * @param user the user performing the action. Use null not to specify - * one. - * @param target the target of the action. Use null not to specify one. - * @param start what timestamp to start. Use null to not specify one. - * @param end what timestamp to end. Use null to not specify one. - * @param amount the amount of log entries to get. If both start and - * end are defined, this is ignored. Use Integer.MAX_VALUE to not - * specify one (overrides global limits) - * @param namespace filters by namespace. Returns empty if namespace - * doesn't exist. Use {@link #ALL_NAMESPACES} to not specify one. + * @param action what action to get (e.g. delete, undelete, etc.), use + * {@code null} to not specify one + * @param helper a {@link Wiki.RequestHelper} (optional, use null to not + * provide any of the optional parameters noted above) * @throws IOException if a network error occurs - * @throws IllegalArgumentException if start < end or amount < 1 + * @throws SecurityException if the user lacks the credentials needed to + * access a privileged log * @return the specified log entries * @since 0.08 */ - public LogEntry[] getLogEntries(String logtype, String action, String user, String target, - OffsetDateTime start, OffsetDateTime end, int amount, int namespace) throws IOException + public List getLogEntries(String logtype, String action, Wiki.RequestHelper helper) throws IOException { - // construct the query url from the parameters given - StringBuilder url = new StringBuilder(query); - url.append("list=logevents&leprop=ids%7Ctitle%7Ctype%7Cuser%7Ctimestamp%7Ccomment%7Cdetails"); - - // check for amount - if (amount < 1) - throw new IllegalArgumentException("Tried to retrieve less than one log entry!"); - + int limit = -1; + Map getparams = new HashMap<>(); + getparams.put("list", "logevents"); + getparams.put("leprop", "ids|title|type|user|timestamp|comment|parsedcomment|details|tags"); if (!logtype.equals(ALL_LOGS)) { if (action == null) - { - url.append("&letype="); - url.append(logtype); - } + getparams.put("letype", logtype); else - { - url.append("&leaction="); - url.append(logtype); - url.append("/"); - url.append(action); - } - } - if (namespace != ALL_NAMESPACES) - { - url.append("&lenamespace="); - url.append(namespace); - } - if (user != null) - { - url.append("&leuser="); - url.append(encode(user, true)); + getparams.put("leaction", logtype + "/" + action); } - if (target != null) + if (helper != null) { - url.append("&letitle="); - url.append(encode(target, true)); - } - if (start != null) - { - if (end != null && start.isBefore(end)) //aargh - throw new IllegalArgumentException("Specified start date is before specified end date!"); - url.append("&lestart="); - url.append(start.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - } - if (end != null) - { - url.append("&leend="); - url.append(end.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + helper.setRequestType("le"); + getparams.putAll(helper.addTitleParameter()); + getparams.putAll(helper.addDateRangeParameters()); + getparams.putAll(helper.addUserParameter()); + getparams.putAll(helper.addReverseParameter()); + getparams.putAll(helper.addNamespaceParameter()); + getparams.putAll(helper.addTagParameter()); + limit = helper.limit(); } - int originallimit = getQueryLimit(); - setQueryLimit(Math.min(amount, originallimit)); - List entries = queryAPIResult("le", url, "getLogEntries", (line, results) -> + List entries = makeListQuery("le", getparams, null, "getLogEntries", limit, (line, results) -> { String[] items = line.split("getLogEntries(), - * getImageHistory() and getIPBlockList() into LogEntry + * Parses xml generated by getLogEntries(), + * getImageHistory() and getBlockList() into {@link Wiki.LogEntry} * objects. Override this if you want custom log types. NOTE: if * RevisionDelete was used on a log entry, the relevant values will be * null. * * @param xml the xml to parse + * @param user null, or use this value for the performer of the log entry + * @param type null, or use this value for the type of the log entry + * @param action null, or use this value for the action of the log entry + * @param target null, or user this value for the target of the log entry * @return the parsed log entry * @since 0.18 */ - protected LogEntry parseLogEntry(String xml) + protected LogEntry parseLogEntry(String xml, String user, String type, String action, String target) { - // note that we can override these in the calling method - String type = "", action = ""; + // ID (getLogEntries only) + long id = -1; + if (xml.contains("logid=\"")) + id = Long.parseLong(parseAttribute(xml, "logid", 0)); + boolean actionhidden = xml.contains("actionhidden=\""); - if (xml.contains("type=\"")) // only getLogEntries + if (type == null && xml.contains("type=\"")) // only getLogEntries { type = parseAttribute(xml, "type", 0); action = parseAttribute(xml, "action", 0); } // reason - String reason; + String reason, parsedreason; boolean reasonhidden = xml.contains("commenthidden=\""); - if (type.equals(USER_CREATION_LOG)) // there is no reason for creating a user + if (USER_CREATION_LOG.equals(type)) + { + // there is no reason for creating a user reason = ""; + parsedreason = ""; + } else if (xml.contains("reason=\"")) + { reason = parseAttribute(xml, "reason", 0); + parsedreason = null; // not available in list=blocks / getBlockList! + } else + { reason = parseAttribute(xml, "comment", 0); + parsedreason = parseAttribute(xml, "parsedcomment", 0); + } - // generic performer name (won't work for ipblocklist, overridden there) + // generic performer name boolean userhidden = xml.contains("userhidden=\"\""); - User performer = null; - if (xml.contains("user=\"")) - performer = new User(parseAttribute(xml, "user", 0)); + if (user == null && xml.contains("user=\"")) + user = parseAttribute(xml, "user", 0); // generic target name - String target = null; - if (xml.contains(" title=\"")) // space is important -- commons.getImageHistory("File:Chief1.gif"); + // space is important -- commons.getImageHistory("File:Chief1.gif"); + if (target == null && xml.contains(" title=\"")) target = parseAttribute(xml, "title", 0); OffsetDateTime timestamp = OffsetDateTime.parse(parseAttribute(xml, "timestamp", 0)); - // details: TODO: make this a HashMap - Object details = null; + // details + Map details = new HashMap<>(); if (xml.contains("commenthidden")) // oversighted details = null; else if (type.equals(MOVE_LOG)) - details = parseAttribute(xml, "target_title", 0); // the new title + details.put("target_title", parseAttribute(xml, "target_title", 0)); else if (type.equals(BLOCK_LOG) || xml.contains(" 10) // not an unblock { int d = s.indexOf('\"', c); - details = new Object[] + if (s.contains("anononly")) // anon-only + details.put("anononly", "true"); + if (s.contains("nocreate")) // account creation blocked + details.put("nocreate", "true"); + if (s.contains("noautoblock")) // autoblock disabled + details.put("noautoblock", "true"); + if (s.contains("noemail")) // email disabled + details.put("noemail", "true"); + if (s.contains("nousertalk")) // cannot edit talk page + details.put("nousertalk", "true"); + details.put("expiry", s.substring(c, d)); + + // partial block parameters + if (s.contains("")) { - s.contains("anononly"), // anon-only - s.contains("nocreate"), // account creation blocked - s.contains("noautoblock"), // autoblock disabled - s.contains("noemail"), // email disabled - s.contains("nousertalk"), // cannot edit talk page - s.substring(c, d) // duration - }; + details.put("partial", "true"); + if (s.contains("")) + { + details.put("type", "namespaces"); + // TODO: add the actual namespaces + } + else if (s.contains("")) + { + details.put("type", "pages"); + // TODO: add the actual pages + } + } } } else if (type.equals(PROTECTION_LOG)) { - if (action.equals("unprotect")) - details = null; - else + if (action.equals("protect")) { // FIXME: return a protectionstate here? - int a = xml.indexOf("") + 7; - int b = xml.indexOf("", a); - details = xml.substring(a, b); + details.put("protection string", parseAttribute(xml, "description", 0)); } } else if (type.equals(USER_RENAME_LOG)) { int a = xml.indexOf("") + 7; int b = xml.indexOf("", a); - details = decode(xml.substring(a, b)); // the new username + details.put("new username", decode(xml.substring(a, b))); } else if (type.equals(USER_RIGHTS_LOG)) { - int a = xml.indexOf("new=\"") + 5; - int b = xml.indexOf('\"', a); - StringTokenizer tk = new StringTokenizer(xml.substring(a, b), ", "); - List temp = new ArrayList<>(); - while (tk.hasMoreTokens()) - temp.add(tk.nextToken()); - details = temp.toArray(new String[temp.size()]); + int a = xml.indexOf(""); + if (a != -1) { + int b = xml.indexOf("", a); + var oldgroups = xml.substring(a + 11, b); + var old_list = new ArrayList(); + for (int end = 0, start = oldgroups.indexOf(""); start != -1; start = oldgroups.indexOf("", end)) { + end = oldgroups.indexOf("", start); + old_list.add(oldgroups.substring(start + 3, end)); + } + details.put("oldgroups", String.join(",", old_list)); + } else // self-closing empty "" tag + details.put("oldgroups", ""); + int c = xml.indexOf(""); + if (c != -1) { + int d = xml.indexOf("", c); + var newgroups = xml.substring(c + 11, d); + var new_list = new ArrayList(); + for (int end = 0, start = newgroups.indexOf(""); start != -1; start = newgroups.indexOf("", end)) { + end = newgroups.indexOf("", start); + new_list.add(newgroups.substring(start + 3, end)); + } + details.put("newgroups", String.join(",", new_list)); + } else // self-closing empty "" tag + details.put("newgroups", ""); + } + + // tags + List tags = new ArrayList<>(); + if (xml.contains("")) + { + for (int idx = xml.indexOf(""); idx >= 0; idx = xml.indexOf("", ++idx)) + tags.add(xml.substring(idx + 5, xml.indexOf("", idx))); } - - LogEntry le = new LogEntry(type, action, reason, performer, target, timestamp, details); - le.userDeleted = userhidden; - le.reasonDeleted = reasonhidden; - le.targetDeleted = actionhidden; + + LogEntry le = new LogEntry(id, timestamp, user, reason, parsedreason, type, action, target, details); + le.setUserDeleted(userhidden); + le.setCommentDeleted(reasonhidden); + le.setContentDeleted(actionhidden); + le.setTags(tags); return le; } @@ -5424,7 +6085,7 @@ else if (type.equals(USER_RIGHTS_LOG)) * @throws IOException if a network error occurs * @since 0.15 */ - public String[] prefixIndex(String prefix) throws IOException + public List prefixIndex(String prefix) throws IOException { return listPages(prefix, null, ALL_NAMESPACES, -1, -1, null); } @@ -5437,7 +6098,7 @@ public String[] prefixIndex(String prefix) throws IOException * @throws IOException if a network error occurs * @since 0.15 */ - public String[] shortPages(int cutoff) throws IOException + public List shortPages(int cutoff) throws IOException { return listPages("", null, MAIN_NAMESPACE, -1, cutoff, null); } @@ -5451,7 +6112,7 @@ public String[] shortPages(int cutoff) throws IOException * @return pages below that size in that namespace * @since 0.15 */ - public String[] shortPages(int cutoff, int namespace) throws IOException + public List shortPages(int cutoff, int namespace) throws IOException { return listPages("", null, namespace, -1, cutoff, null); } @@ -5464,7 +6125,7 @@ public String[] shortPages(int cutoff, int namespace) throws IOException * @throws IOException if a network error occurs * @since 0.15 */ - public String[] longPages(int cutoff) throws IOException + public List longPages(int cutoff) throws IOException { return listPages("", null, MAIN_NAMESPACE, cutoff, -1, null); } @@ -5478,7 +6139,7 @@ public String[] longPages(int cutoff) throws IOException * @throws IOException if a network error occurs * @since 0.15 */ - public String[] longPages(int cutoff, int namespace) throws IOException + public List longPages(int cutoff, int namespace) throws IOException { return listPages("", null, namespace, cutoff, -1, null); } @@ -5500,7 +6161,7 @@ public String[] longPages(int cutoff, int namespace) throws IOException * @since 0.09 * @throws IOException if a network error occurs */ - public String[] listPages(String prefix, Map protectionstate, int namespace) throws IOException + public List listPages(String prefix, Map protectionstate, int namespace) throws IOException { return listPages(prefix, protectionstate, namespace, -1, -1, null); } @@ -5518,7 +6179,7 @@ public String[] listPages(String prefix, Map protectionstate, in * @param protectionstate a {@link #protect protection state}, use null * to not specify one * @param namespace a namespace. ALL_NAMESPACES is not suppported, an - * UnsupportedOperationException will be thrown. + * UnsupportedOperationException will be thrown unless a prefix is specified. * @param minimum the minimum size in bytes these pages can be. Use -1 to * not specify one. * @param maximum the maximum size in bytes these pages can be. Use -1 to @@ -5529,116 +6190,96 @@ public String[] listPages(String prefix, Map protectionstate, in * @since 0.09 * @throws IOException if a network error occurs */ - public String[] listPages(String prefix, Map protectionstate, int namespace, int minimum, + public List listPages(String prefix, Map protectionstate, int namespace, int minimum, int maximum, Boolean redirects) throws IOException { - // @revised 0.15 to add short/long pages // No varargs namespace here because MW API only supports one namespace // for this module. - StringBuilder url = new StringBuilder(query); - url.append("list=allpages"); - if (!prefix.isEmpty()) // prefix + Map getparams = new HashMap<>(); + getparams.put("list", "allpages"); + if (!prefix.isEmpty()) { - // cull the namespace prefix - namespace = namespace(prefix); - if (prefix.contains(":") && namespace != MAIN_NAMESPACE) - prefix = prefix.substring(prefix.indexOf(':') + 1); - url.append("&apprefix="); - url.append(encode(prefix, true)); + if (namespace == ALL_NAMESPACES) + { + namespace = namespace(prefix); + prefix = removeNamespace(prefix); + } + getparams.put("apprefix", normalize(prefix)); } else if (namespace == ALL_NAMESPACES) // check for namespace throw new UnsupportedOperationException("ALL_NAMESPACES not supported in MediaWiki API."); - url.append("&apnamespace="); - url.append(namespace); + getparams.put("apnamespace", String.valueOf(namespace)); if (protectionstate != null) { - StringBuilder apprtype = new StringBuilder("&apprtype="); - StringBuilder apprlevel = new StringBuilder("&apprlevel="); - for (Map.Entry entry : protectionstate.entrySet()) + StringBuilder apprtype = new StringBuilder(); + StringBuilder apprlevel = new StringBuilder(); + protectionstate.forEach((key, value) -> { - String key = entry.getKey(); if (key.equals("cascade")) - { - url.append("&apprfiltercascade="); - url.append((Boolean)entry.getValue() ? "cascading" : "noncascading"); - } + getparams.put("apprfiltercascade", (Boolean)value ? "cascading" : "noncascading"); else if (!key.contains("expiry")) { apprtype.append(key); - apprtype.append("%7C"); - apprlevel.append((String)entry.getValue()); - apprlevel.append("%7C"); + apprtype.append("|"); + apprlevel.append(value); + apprlevel.append("|"); } - } - apprtype.delete(apprtype.length() - 3, apprtype.length()); - apprlevel.delete(apprlevel.length() - 3, apprlevel.length()); - url.append(apprtype); - url.append(apprlevel); + }); + getparams.put("apprtype", apprtype.substring(0, apprtype.length() - 1)); + getparams.put("apprlevel", apprlevel.substring(0, apprlevel.length() - 1)); } // max and min - if (minimum != -1) - { - url.append("&apminsize="); - url.append(minimum); - } - if (maximum != -1) - { - url.append("&apmaxsize="); - url.append(maximum); - } + if (minimum >= 0) + getparams.put("apminsize", String.valueOf(minimum)); + if (maximum >= 0) + getparams.put("apmaxsize", String.valueOf(maximum)); if (redirects != null) - { - url.append("&apfilterredir="); - url.append(redirects ? "redirects" : "nonredirects"); - } + getparams.put("apfilterredir", redirects ? "redirects" : "nonredirects"); // set query limit = 1 request if max, min, prefix or protection level // not specified - int originallimit = getQueryLimit(); + int limit = -1; if (maximum < 0 && minimum < 0 && prefix.isEmpty() && protectionstate == null) - setQueryLimit(max); - List pages = queryAPIResult("ap", url, "listPages", (line, results) -> + limit = max; + List pages = makeListQuery("ap", getparams, null, "listPages", limit, (line, results) -> { // xml form:

for (int a = line.indexOf("

0; a = line.indexOf("

Warnings: + *

+ * + * @param page one of the qppage values specifed by the documentation below + * (case sensitive) * @return the list of pages returned by that particular special page * @throws IOException if a network error occurs - * @throws CredentialNotFoundException if page=Unwatchedpages and we cannot - * read it + * @throws SecurityException if the user lacks the privileges necessary to + * view a report (e.g. unwatchedpages) * @since 0.28 + * @see MediaWiki + * documentation */ - public String[] queryPage(String page) throws IOException, CredentialNotFoundException + public List queryPage(String page) throws IOException { - if (page.equals("Unwatchedpages") && (user == null || !user.isAllowedTo("unwatchedpages"))) - throw new CredentialNotFoundException("User does not have the \"unwatchedpages\" permission."); + Map getparams = new HashMap<>(); + getparams.put("list", "querypage"); + getparams.put("qppage", page); - StringBuilder url = new StringBuilder(query); - url.append("action=query&list=querypage&qppage="); - url.append(page); - - List pages = queryAPIResult("qp", url, "queryPage", (line, results) -> + List pages = makeListQuery("qp", getparams, null, "queryPage", -1, (line, results) -> { // xml form: for (int x = line.indexOf(" 0; x = line.indexOf("amount most recently created pages in the main - * namespace. WARNING: The - * recentchanges table stores new pages for a finite period of time; - * it is not possible to retrieve pages created before then. + * Fetches recently created pages. See {@link #recentChanges(Wiki.RequestHelper, + * String)} for full documentation. Equivalent to [[Special:Newpages]]. * - * @param amount the number of pages to fetch (overrides global query - * limits) - * @return the revisions that created the pages satisfying the requirements - * above - * @throws IOException if a network error occurs - * @since 0.20 - */ - public Revision[] newPages(int amount) throws IOException - { - return recentChanges(amount, 0, true, MAIN_NAMESPACE); - } - - /** - * Fetches the amount most recently created pages in the main - * namespace subject to the specified constraints. WARNING: The recentchanges - * table stores new pages for a finite period of - * time; it is not possible to retrieve pages created before then. - * Equivalent to [[Special:Newpages]]. - * - * @param rcoptions a bitmask of {@link #HIDE_ANON} etc that dictate which - * pages we return (e.g. to exclude patrolled pages set rcoptions = HIDE_PATROLLED). - * @param amount the amount of new pages to get (overrides global query - * limits) - * @param ns a list of namespaces to filter by, empty = all namespaces. + * @param helper a {@link Wiki.RequestHelper} (optional, use null to not + * provide any optional parameters * @return the revisions that created the pages satisfying the requirements * above * @throws IOException if a network error occurs - * @since 0.20 + * @since 0.35 */ - public Revision[] newPages(int amount, int rcoptions, int... ns) throws IOException + public List newPages(Wiki.RequestHelper helper) throws IOException { - return recentChanges(amount, rcoptions, true, ns); + return recentChanges(helper, "new"); } /** - * Fetches the amount most recent changes in the main namespace. - * WARNING: The - * recentchanges table stores new pages for a finite period of - * time; it is not possible to retrieve pages created before then. + * Fetches recent edits to this wiki. See {@link + * #recentChanges(Wiki.RequestHelper, String)} for full documentation. * Equivalent to [[Special:Recentchanges]]. - *

- * Note: Log entries in recent changes have a revid of 0! * - * @param amount the number of entries to return (overrides global query - * limits) + * @param helper a {@link Wiki.RequestHelper} (optional, use null to not + * provide any optional parameters * @return the recent changes that satisfy these criteria * @throws IOException if a network error occurs * @since 0.23 */ - public Revision[] recentChanges(int amount) throws IOException + public List recentChanges(Wiki.RequestHelper helper) throws IOException { - return recentChanges(amount, 0, false, MAIN_NAMESPACE); + return recentChanges(helper, null); } /** - * Fetches the amount most recent changes in the specified - * namespace. WARNING: The recent changes table only stores new pages for - * about a month. It is not possible to retrieve changes before then. - * Equivalent to [[Special:Recentchanges]]. + * Fetches recent changes to this wiki. WARNING: The recentchanges + * table stores edits for a + * finite period of time; it is not possible to retrieve pages created + * before then. Equivalent to [[Special:Recentchanges]]. + * *

- * Note: Log entries in recent changes have a revid of 0! + * Accepted parameters from helper are: + *

    + *
  • {@link Wiki.RequestHelper#withinDateRange(OffsetDateTime, + * OffsetDateTime) date range} + *
  • {@link Wiki.RequestHelper#byUser(String) user} + *
  • {@link Wiki.RequestHelper#notByUser(String) not by user} + *
  • {@link Wiki.RequestHelper#reverse(boolean) reverse} + *
  • {@link Wiki.RequestHelper#inNamespaces(int...) namespaces} + *
  • {@link Wiki.RequestHelper#taggedWith(String) tag} + *
  • {@link Wiki.RequestHelper#filterBy(Map) filter by}: "minor", "bot", + * "anon", "redirect", "patrolled" + *
  • {@link Wiki.RequestHelper#limitedTo(int) local query limit} + *
* - * @param amount the number of entries to return (overrides global query - * limits) - * @param ns a list of namespaces to filter by, empty = all namespaces. - * @return the recent changes that satisfy these criteria - * @throws IOException if a network error occurs - * @since 0.23 - */ - public Revision[] recentChanges(int amount, int[] ns) throws IOException - { - return recentChanges(amount, 0, false, ns); - } - - /** - * Fetches the amount most recent changes in the specified - * namespace subject to the specified constraints. WARNING: The recent - * changes table only stores new pages for about a month. It is not - * possible to retrieve changes before then. Equivalent to - * [[Special:Recentchanges]]. *

- * Note: Log entries in recent changes have a revid of 0! + * If {@code rctype} is not {@code "edit"} or {@code "new"} then the results + * consist of pseudo-revisions whose data does not correspond to an actual + * on-wiki state. For example: * - * @param amount the number of entries to return (overrides global query - * limits) - * @param ns a list of namespaces to filter by, empty = all namespaces. - * @param rcoptions a bitmask of HIDE_ANON etc that dictate which pages - * we return. + *

    + *
  • {@code rctype == "log"} yields {@code id == 0} and {@code title} is + * the log entry target + *
  • {@code rctype == "external"} yields {@code id} as the most recent + * edit to {@code title}, {@code previous_id == id}, {@code user} is + * the external user making the change, {@code sizediff == 0} and + * {@code comment} describes the external change + *
  • {@code rctype =="categorize"} yields {@code title} as the category + * added or removed and {@code comment} specifies the page added or + * removed to that category + *
+ * + * @param helper a {@link Wiki.RequestHelper} (optional, use null to not + * provide any of the optional parameters described above + * @param rctype null, "edit" (edits only) or "new" (new pages); your + * mileage may vary for other types (log, external, categorize) * @return the recent changes that satisfy these criteria * @throws IOException if a network error occurs - * @since 0.23 + * @since 0.35 */ - public Revision[] recentChanges(int amount, int rcoptions, int... ns) throws IOException + protected List recentChanges(Wiki.RequestHelper helper, String rctype) throws IOException { - return recentChanges(amount, rcoptions, false, ns); - } + int limit = -1; + Map getparams = new HashMap<>(); + getparams.put("list", "recentchanges"); + getparams.put("rcprop", "title|ids|user|timestamp|flags|comment|parsedcomment|sizes|sha1|tags"); + if (helper != null) + { + helper.setRequestType("rc"); + getparams.putAll(helper.addNamespaceParameter()); + getparams.putAll(helper.addUserParameter()); + getparams.putAll(helper.addExcludeUserParameter()); + getparams.putAll(helper.addDateRangeParameters()); + getparams.putAll(helper.addTagParameter()); + getparams.putAll(helper.addReverseParameter()); + getparams.putAll(helper.addShowParameter()); + limit = helper.limit(); + } - /** - * Fetches the amount most recent changes in the specified - * namespace subject to the specified constraints. WARNING: The recent - * changes table only stores new pages for about a month. It is not - * possible to retrieve changes before then. Equivalent to - * [[Special:Recentchanges]]. - *

- * Note: Log entries in recent changes have a revid of 0! - * - * @param amount the number of entries to return (overrides global - * query limits) - * @param ns a list of namespaces to filter by, empty = all namespaces. - * @param rcoptions a bitmask of HIDE_ANON etc that dictate which pages - * we return. - * @param newpages show new pages only - * @return the recent changes that satisfy these criteria - * @throws IOException if a network error occurs - * @since 0.23 - */ - protected Revision[] recentChanges(int amount, int rcoptions, boolean newpages, int... ns) throws IOException - { - StringBuilder url = new StringBuilder(query); - url.append("list=recentchanges&rcprop=title%7Cids%7Cuser%7Ctimestamp%7Cflags%7Ccomment%7Csizes%7Csha1"); - constructNamespaceString(url, "rc", ns); - if (newpages) - url.append("&rctype=new"); - // rc options - if (rcoptions > 0) - { - url.append("&rcshow="); - if ((rcoptions & HIDE_ANON) == HIDE_ANON) - url.append("!anon%7C"); - if ((rcoptions & HIDE_SELF) == HIDE_SELF) - url.append("!self%7C"); - if ((rcoptions & HIDE_MINOR) == HIDE_MINOR) - url.append("!minor%7C"); - if ((rcoptions & HIDE_PATROLLED) == HIDE_PATROLLED) - url.append("!patrolled%7C"); - if ((rcoptions & HIDE_BOT) == HIDE_BOT) - url.append("!bot%7C"); - // chop off last | - url.delete(url.length() - 3, url.length()); - } - - int originallimit = getQueryLimit(); - setQueryLimit(amount); - List revisions = queryAPIResult("rc", url, newpages ? "newPages" : "recentChanges", + if (rctype != null) + getparams.put("rctype", rctype); + + List revisions = makeListQuery("rc", getparams, null, "recentChanges", limit, (line, results) -> { // xml form for (int i = line.indexOf(" 0; i = line.indexOf("", i); + int j = line.indexOf("", i); results.add(parseRevision(line.substring(i, j), "")); } }); - setQueryLimit(originallimit); - int temp = revisions.size(); - log(Level.INFO, "recentChanges", "Successfully retrieved recent changes (" + temp + " revisions)"); - return revisions.toArray(new Revision[temp]); + log(Level.INFO, "recentChanges", "Successfully retrieved recent changes (" + revisions.size() + " revisions)"); + return revisions; } /** * Fetches all pages that use interwiki links to the specified wiki and the - * page on that wiki that is linked to. For example, - * getInterWikiBacklinks("testwiki") may return: + * page on that wiki that is linked to. For example, {@code + * getInterWikiBacklinks("testwiki")} may return: * * { * { "Spam", "testwiki:Blah" }, @@ -5844,7 +6431,7 @@ protected Revision[] recentChanges(int amount, int rcoptions, boolean newpages, * @throws IOException if a network error occurs * @since 0.23 */ - public String[][] getInterWikiBacklinks(String prefix) throws IOException + public List getInterWikiBacklinks(String prefix) throws IOException { return getInterWikiBacklinks(prefix, "|"); } @@ -5858,8 +6445,8 @@ public String[][] getInterWikiBacklinks(String prefix) throws IOException * *

* Example: If [[Test]] and [[Spam]] both contain the interwiki link - * [[testwiki:Blah]] then getInterWikiBacklinks("testwiki", "Blah"); - * will return (sorted by title) + * [[testwiki:Blah]] then {@code getInterWikiBacklinks("testwiki", "Blah");} + * will return (sorted by title) * * { * { "Spam", "testwiki:Blah" }, @@ -5882,23 +6469,20 @@ public String[][] getInterWikiBacklinks(String prefix) throws IOException * prefix (the MediaWiki API doesn't like this) * @since 0.23 */ - public String[][] getInterWikiBacklinks(String prefix, String title) throws IOException + public List getInterWikiBacklinks(String prefix, String title) throws IOException { // must specify a prefix if (title.equals("|") && prefix.isEmpty()) throw new IllegalArgumentException("Interwiki backlinks: title specified without prefix!"); - StringBuilder url = new StringBuilder(query); - url.append("list=iwbacklinks&iwblprefix="); - url.append(prefix); + Map getparams = new HashMap<>(); + getparams.put("list", "iwbacklinks"); + getparams.put("iwblprefix", prefix); if (!title.equals("|")) - { - url.append("&iwbltitle="); - url.append(title); - } - url.append("&iwblprop=iwtitle%7Ciwprefix"); + getparams.put("iwbltitle", normalize(title)); + getparams.put("iwblprop", "iwtitle|iwprefix"); - List links = queryAPIResult("iwbl", url, "getInterWikiBacklinks", (line, results) -> + List links = makeListQuery("iwbl", getparams, null, "getInterWikiBacklinks", -1, (line, results) -> { // xml form: for (int x = line.indexOf(" 0; x = line.indexOf(" { - private String username; - private String[] rights = null; // cache - private String[] groups = null; // cache + private final String username; + private final OffsetDateTime registration; + // user privileges (volatile, changes rarely) + private List rights; + private List groups; + private boolean blocked; + // user preferences (volatile, changes rarely) + private Gender gender; + private boolean emailable; + // volatile, changes often + private int editcount; /** * Creates a new user object. Does not create a new user on the @@ -5933,11 +6525,27 @@ public class User implements Cloneable, Serializable * be called for anons. * * @param username the username of the user + * @param registration when the user was registered + * @param rights the rights this user has + * @param groups the groups this user belongs to + * @param gender the self-declared {@link Wiki.Gender Gender} of this user. + * @param emailable whether the user can be emailed through [[Special:Emailuser]] + * @param blocked whether this user is blocked + * @param editcount the internal edit count of this user * @since 0.05 */ - protected User(String username) + protected User(String username, OffsetDateTime registration, List rights, List groups, + Gender gender, boolean emailable, boolean blocked,int editcount) { - this.username = username; + this.username = Objects.requireNonNull(username); + // can be null per https://phabricator.wikimedia.org/T24097 + this.registration = registration; + this.rights = Objects.requireNonNull(rights); + this.groups = Objects.requireNonNull(groups); + this.gender = gender; + this.emailable = emailable; + this.blocked = blocked; + this.editcount = editcount; } /** @@ -5945,111 +6553,131 @@ protected User(String username) * @return this user's username * @since 0.08 */ - public String getUsername() + public final String getUsername() { return username; } /** - * Gets various properties of this user. Returns: - *

    - *
  • editcount: (int) {@link #countEdits()} the user's edit - * count - *
  • groups: (String[]) the groups the user is in (see - * [[Special:Listgrouprights]]) - *
  • rights: (String[]) the stuff the user can do - *
  • emailable: (Boolean) whether the user can be emailed - * through [[Special:Emailuser]] or emailUser() - *
  • blocked: (Boolean) whether the user is blocked - *
  • gender: (Wiki.Gender) the user's gender - *
  • created: (Calendar) when the user account was created - *
- * + * Gets the date/time at which this user account was created. May be + * {@code null} per + * https://phabricator.wikimedia.org/T24097. * @return (see above) - * @throws IOException if a network error occurs - * @since 0.24 + * @since 0.35 */ - public Map getUserInfo() throws IOException + public final OffsetDateTime getRegistrationDate() { - return Wiki.this.getUserInfo(new String[] { username })[0]; + return registration; } /** - * Returns true if the user is allowed to perform the specified action. - * Uses the rights cache. Read [[Special:Listgrouprights]] before using - * this! + * Returns {@code true} if the user is allowed to perform the specified + * action(s). Read [[Special:Listgrouprights]] before using this! * @param right a specific action - * @return whether the user is allowed to execute it + * @param morerights additional actions to check + * @return whether the user is allowed to execute them * @since 0.24 - * @throws IOException if a network error occurs */ - public boolean isAllowedTo(String right) throws IOException - { - // We can safely assume the user is allowed to { read, edit, create, - // writeapi }. - if (rights == null) - rights = (String[])getUserInfo().get("rights"); - for (String r : rights) - if (r.equals(right)) - return true; - return false; + public boolean isAllowedTo(String right, String... morerights) + { + List temp = new ArrayList<>(); + temp.add(right); + temp.addAll(List.of(morerights)); + return rights.containsAll(temp); } /** - * Returns true if the user is a member of the specified group. Uses - * the groups cache. + * Returns {@code true} if the user is a member of the specified group. * @param group a specific group * @return whether the user is in it * @since 0.24 - * @throws IOException if a network error occurs */ - public boolean isA(String group) throws IOException + public boolean isA(String group) { - if (groups == null) - groups = (String[])getUserInfo().get("groups"); - for (String g : groups) - if (g.equals(group)) - return true; - return false; + return groups.contains(group); } /** - * Returns a log of the times when the user has been blocked. - * @return records of the occasions when this user has been blocked - * @throws IOException if something goes wrong - * @since 0.08 + * Returns the groups the user is a member of. See [[Special:Listgrouprights]]. + * Changes in this list do not propagate to this object or the wiki. + * @return (see above) + * @since 0.35 + */ + public List getGroups() + { + return new ArrayList<>(groups); + } + + /** + * Returns the specific permissions this user has. See [[Special:Listgrouprights]]. + * Changes in this list do not propagate to the object or the wiki. + * @return (see above) + * @since 0.35 + */ + public List getRights() + { + return new ArrayList<>(rights); + } + + /** + * Returns whether this user can be emailed through [[Special:Emailuser]]. + * @return (see above) + * @see #emailUser(Wiki.User, String, String, boolean) + * @since 0.35 + */ + public boolean canBeEmailed() + { + return emailable; + } + + /** + * Returns the self-disclosed {@linkplain Wiki.Gender gender} of this + * user. + * @return (see above) + * @see Wiki.Gender + * @since 0.35 */ - public LogEntry[] blockLog() throws IOException + public Gender getGender() { - return Wiki.this.getLogEntries(Wiki.BLOCK_LOG, null, null, "User:" + username); + return gender; } /** - * Determines whether this user is blocked by looking it up on the IP - * block list. + * Determines whether this user is blocked at the time of construction. + * If you want a live check, look up the user on the {@linkplain + * #getBlockList list of blocks}. * @return whether this user is blocked - * @throws IOException if we cannot retrieve the IP block list * @since 0.12 */ - public boolean isBlocked() throws IOException + public boolean isBlocked() { - // @revised 0.18 now check for errors after each edit, including blocks - return getIPBlockList(username, null, null).length != 0; + return blocked; } /** - * Fetches the internal edit count for this user, which includes all - * live edits and deleted edits after (I think) January 2007. If you - * want to count live edits only, use the slower - * int count = {@link User#contribs(int...) user.contribs()}.length;. + * Fetches the internal edit count for this user at the time of + * construction, which includes all live edits and deleted edits after + * (I think) January 2007. If you want to count live edits only, + * compute the size of {@link User#contribs(int...) User.contribs()}. * * @return the user's edit count - * @throws IOException if a network error occurs * @since 0.16 */ - public int countEdits() throws IOException + public int countEdits() { - return (Integer)getUserInfo().get("editcount"); + return editcount; + } + + /** + * Returns a log of the times when the user has been blocked. + * @return records of the occasions when this user has been blocked + * @throws IOException if something goes wrong + * @since 0.08 + */ + public List blockLog() throws IOException + { + Wiki.RequestHelper rh = new RequestHelper().byTitle("User:" + username); + return Wiki.this.getLogEntries(Wiki.BLOCK_LOG, null, rh); } /** @@ -6059,243 +6687,430 @@ public int countEdits() throws IOException * @throws IOException if a network error occurs * @since 0.17 */ - public Revision[] contribs(int... ns) throws IOException + public List contribs(int... ns) throws IOException { - return Wiki.this.contribs(username, ns); + Wiki.RequestHelper rh = new RequestHelper().inNamespaces(ns); + return Wiki.this.contribs(username, rh); } - + /** * Returns the list of logged actions performed by this user. * @param logtype what log to get ({@link Wiki#DELETION_LOG} etc.) - * @param action what action to get (e.g. delete, undelete), use + * @param action what action to get (e.g. delete, undelete), use * "" to not specify one * @return (see above) * @throws IOException if a network error occurs * @since 0.33 */ - public LogEntry[] getLogEntries(String logtype, String action) throws IOException + public List getLogEntries(String logtype, String action) throws IOException { - return Wiki.this.getLogEntries(logtype, action, username, null); + Wiki.RequestHelper rh = new RequestHelper().byUser(username); + return Wiki.this.getLogEntries(logtype, action, rh); } /** - * Copies this user object. - * @return the copy - * @throws CloneNotSupportedException if the clone fails - * @since 0.08 + * Tests whether this user is equal to another one. + * @param x another object + * @return whether the usernames of the users are equal */ @Override - public User clone() throws CloneNotSupportedException + public boolean equals(Object x) { - try - { - return (User)super.clone(); - } - catch (CloneNotSupportedException e) - { - return null; - } + if (!(x instanceof User)) + return false; + User other = (User)x; + return Objects.equals(username, other.username) + && Objects.equals(registration, other.registration); } /** - * Tests whether this user is equal to another one. - * @param x another object - * @return whether the usernames of the users are equal - * @since 0.08 + * Returns a hashcode of this user based on the username and + * registration date. + * @return see above */ @Override - public boolean equals(Object x) + public int hashCode() { - return x instanceof User && username.equals(((User)x).username); + return username.hashCode() * 127 + registration.hashCode(); } /** - * Returns a string representation of this user. - * @return see above - * @since 0.17 + * Enables sorting of users by their username. + * @param other some other user + * @return less than zero if this user is alphabetically before the + * other, 0 if they are the same and 1 if alphabetically after */ @Override - public String toString() + public int compareTo(User other) { - StringBuilder temp = new StringBuilder("User[username="); - temp.append(username); - temp.append("groups="); - temp.append(groups != null ? Arrays.toString(groups) : "unset"); - temp.append("]"); - return temp.toString(); + return username.compareTo(other.username); } /** - * Returns a hashcode of this user. + * Returns a string representation of this user. * @return see above - * @since 0.19 */ @Override - public int hashCode() + public String toString() { - return username.hashCode() * 2 + 1; + return getClass().getName() + + "[username=" + username + + ",registration=" + (registration != null ? registration.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) : "unset") + + ",groups=" + Arrays.toString(groups.toArray()) + "]"; } } /** - * A wrapper class for an entry in a wiki log, which represents an action - * performed on the wiki. - * - * @see #getLogEntries - * @since 0.08 + * A data super class for an event happening on a wiki such as a {@link + * Wiki.Revision} or a {@link Wiki.LogEntry}. + * @since 0.35 */ - public class LogEntry implements Comparable + public abstract class Event implements Comparable { - // internal data storage - private long logid = -1; - private String type; - private String action; - private final String reason; - private User user; - private String target; + private final long id; private final OffsetDateTime timestamp; - private Object details; - private boolean reasonDeleted = false, userDeleted = false, - targetDeleted = false; + private final String user; + private final String title; + private final String comment; + private final String parsedcomment; + private List tags; + private boolean commentDeleted = false, userDeleted = false, + contentDeleted = false; /** - * Creates a new log entry. WARNING: does not perform the action - * implied. Use Wiki.class methods to achieve this. - * - * @param type the type of log entry, one of {@link #USER_CREATION_LOG}, - * {@link #DELETION_LOG}, {@link #BLOCK_LOG}, etc. - * @param action the type of action that was performed e.g. "delete", - * "unblock", "overwrite", etc. - * @param reason why the action was performed - * @param user the user who performed the action - * @param target the target of the action - * @param timestamp the local time when the action was performed. - * We will convert this back into an OffsetDateTime - * @param details the details of the action (e.g. the new title of - * the page after a move was performed). - * @since 0.08 + * Creates a new Event record. + * @param id the unique ID of the event + * @param timestamp the timestamp at which it occurred + * @param user the user or IP address performing the event + * @param title the title of the page affected + * @param comment the comment left by the user when performing the + * event (e.g. an edit summary) + * @param parsedcomment comment, but parsed into HTML */ - protected LogEntry(String type, String action, String reason, User user, - String target, OffsetDateTime timestamp, Object details) + protected Event(long id, OffsetDateTime timestamp, String user, String title, String comment, String parsedcomment) { - this.type = type; - this.action = action; - this.reason = reason; + this.id = id; + this.timestamp = Objects.requireNonNull(timestamp); this.user = user; - this.target = target; - this.timestamp = timestamp; - this.details = details; + this.title = title; + this.comment = comment; + // Rewrite parsedcomments to fix useless relative hyperlinks to + // other wiki pages + if (parsedcomment == null) + this.parsedcomment = null; + else + this.parsedcomment = parsedcomment.replace("href=\"/wiki", "href=\"" + protocol + domain + "/wiki"); } - + + /** + * Gets the unique ID of this event. For a {@link Wiki.Revision}, this + * number is referred to as the "oldid" or "revid" and should not be + * confused with "rcid" (which is the ID in the recentchanges table). + * For a {@link Wiki.LogEntry}, this value only makes sense if the + * record was obtained through {@link Wiki#getLogEntries(String, String, + * RequestHelper)} and overloads (other methods return pseudo-LogEntries). + * @return the ID of this revision + */ + public long getID() + { + return id; + } + + /** + * Gets the timestamp of this event. + * @return the timestamp of this event + */ + public OffsetDateTime getTimestamp() + { + return timestamp; + } + /** - * Gets the ID of this log entry. Only available if retrieved by - * {@link Wiki#getLogEntries}, otherwise returns -1. + * Returns the user or anon who performed this event. You should pass + * this (if not an IP) to {@link #getUser} to obtain a {@link + * Wiki.User} object. Returns {@code null} if the user was + * RevisionDeleted and you lack the necessary privileges. + * @return the user or anon + */ + public String getUser() + { + return user; + } + + /** + * Sets a boolean flag that the user triggering this event has been + * RevisionDeleted in on-wiki records. + * @param deleted (see above) + * @see #isUserDeleted() + * @see #getUser() + */ + protected void setUserDeleted(boolean deleted) + { + userDeleted = deleted; + } + + /** + * Returns {@code true} if the user triggering this event is + * RevisionDeleted. * @return (see above) - * @since 0.33 + * @see #getUser() */ - public long getLogID() + public boolean isUserDeleted() { - return logid; + return userDeleted; } /** - * Gets the type of log that this entry is in. - * @return one of {@link Wiki#DELETION_LOG}, {@link Wiki#BLOCK_LOG}, etc. - * @since 0.08 + * Returns the page affected by this event. May be {@code null} for + * certain types of LogEntry and/or if the LogEntry is RevisionDeleted + * and you don't have the ability to access it. + * @return (see above) + * @see #isContentDeleted() */ - public String getType() + public String getTitle() { - return type; + return title; } /** - * Gets a string description of the action performed, for example - * "delete", "protect", "overwrite", ... WARNING: returns null if the - * action was RevisionDeleted. - * @return the type of action performed - * @since 0.08 + * Gets the comment for this event in wikitext. If this is a {@link + * Wiki.Revision}, this is the edit summary. If this is a {@link + * Wiki.LogEntry}, this is the reason for the logged action. WARNING: + * returns {@code null} if the reason was RevisionDeleted and you lack + * the necessary privileges. + * @return the comment associated with the event + * @see #getParsedComment() */ - public String getAction() + public String getComment() { - return action; + return comment; } - + + /** + * Gets the comment for this event, with limited parsing into HTML. + * Hyperlinks in the returned HTML are rewritten from useless relative + * URLs to full URLs that point to the wiki page in question. Returns + * {@code null} if {@linkplain #isCommentDeleted() the comment was + * RevisionDeleted} and you lack the necessary privileges. + * + *

Warnings: + *

    + *
  • Not available through {@link #getBlockList}. + *
+ * + * @return the comment associated with the event, parsed into HTML + * @see #getComment() + */ + public String getParsedComment() + { + return parsedcomment; + } + /** - * Returns true if the target has been RevisionDeleted (action is hidden - * in the GUI but retrievable by the API). + * Sets a boolean flag that the comment associated with this event has + * been RevisionDeleted in on-wiki records. + * @param deleted (see above) + * @see #getComment + * @see #getParsedComment() + * @see #isCommentDeleted() + */ + protected void setCommentDeleted(boolean deleted) + { + commentDeleted = deleted; + } + + /** + * Returns {@code true} if the comment is RevisionDeleted. * @return (see above) - * @since 0.32 + * @see #getComment + * @see #getParsedComment() */ - public boolean isTargetDeleted() + public boolean isCommentDeleted() { - return targetDeleted; + return commentDeleted; } /** - * Gets the reason supplied by the perfoming user when the action - * was performed. WARNING: returns null if the reason was - * RevisionDeleted and one does not have access to the content. - * @return the reason the action was performed - * @since 0.08 + * Sets a boolean flag that the content of this event has been + * RevisionDeleted. + * @param deleted (see above) + * @see #isContentDeleted() + */ + protected void setContentDeleted(boolean deleted) + { + contentDeleted = deleted; + } + + /** + * Returns {@code true} if the content of this event has been + * RevisionDeleted. For a {@link Wiki.LogEntry}, this refers to the + * page the logged action affects and the logged action performed (e.g. + * "unblock" or "delete"). + * @return (see above) */ - public String getReason() + public boolean isContentDeleted() { - return reason; + return contentDeleted; } /** - * Returns true if the reason is RevisionDeleted. + * Returns the list of tags attached to this event. Modifying the + * return value does not affect this Revision object or the wiki state. * @return (see above) - * @since 0.32 + * @see MediaWiki + * documentation + * @since 0.37 */ - public boolean isReasonDeleted() + public List getTags() { - return reasonDeleted; + return new ArrayList<>(tags); + } + + /** + * Sets the list of tags attached to this event. Modifying the supplied + * list does not affect this Revision object or change on-wiki state. + * @param tags a list of change tags + * @see MediaWiki + * documentation + * @since 0.37 + */ + protected void setTags(List tags) + { + this.tags = new ArrayList<>(tags); } /** - * Gets the user object representing who performed the action. - * WARNING: returns null if the user was RevisionDeleted and one does - * not have access to the content. - * @return the user who performed the action. - * @since 0.08 + * Returns a String representation of this Event. Subclasses only need + * to lop off the trailing "]" and add their own fields when overriding + * this method. + * @return (see above) + */ + @Override + public String toString() + { + return getClass().getName() + + "[id=" + id + + ",timestamp=" + timestamp.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME) + + ",user=\"" + Objects.toString(user, "[DELETED]") + "\"" + + ",userDeleted=" + userDeleted + + ",title=\"" + Objects.toString(title, "[null or deleted]") + "\"" + + ",comment=\"" + Objects.toString(comment, "[DELETED]") + "\"" + + ",commentDeleted=" + commentDeleted + + ",contentDeleted=" + contentDeleted + ']'; + } + + /** + * Determines whether this Event is equal to some other object. This + * method checks the ID, timestamp, user, title and comment. + * @param other the other object to compare to + * @return whether this instance is equal to that object */ - public User getUser() + @Override + public boolean equals(Object other) { - return user; + if (!(other instanceof Event)) + return false; + Event event = (Event)other; + return id == event.id + && Objects.equals(timestamp, event.timestamp) + && Objects.equals(user, event.user) + && Objects.equals(title, event.title) + && Objects.equals(comment, event.comment); } - + /** - * Returns true if the user who performed this LogEntry is - * RevisionDeleted. + * Returns a hash code for this object based on the ID, timestamp, + * user, title and comment. * @return (see above) - * @since 0.32 */ - public boolean isUserDeleted() + @Override + public int hashCode() { - return userDeleted; + int hc = Long.hashCode(id); + hc = 127 * hc + timestamp.hashCode(); + hc = 127 * hc + Objects.hashCode(user); + hc = 127 * hc + Objects.hashCode(title); + hc = 127 * hc + Objects.hashCode(comment); + return hc; + } + + /** + * Compares this event to another one based on the recentness of their + * timestamps (more recent = positive return value), then + * alphabetically by user. + * @param other the event to compare to + * @return the comparator value, negative if less, positive if greater + */ + @Override + public int compareTo(Wiki.Event other) + { + int result = timestamp.compareTo(other.timestamp); + if (result == 0 && user != null) + result = user.compareTo(other.user); + return result; + } + } + + /** + * A wrapper class for an entry in a wiki log, which represents an action + * performed on the wiki. + * + * @see #getLogEntries + * @since 0.08 + */ + public class LogEntry extends Event + { + private final String type; + private String action; + private Map details; + + /** + * Creates a new log entry. WARNING: does not perform the action + * implied. Use Wiki.class methods to achieve this. + * + * @param id the unique of this log entry + * @param timestamp the local time when the action was performed. + * @param user the user who performed the action + * @param comment why the action was performed + * @param parsedcomment like comment, but parsed into HTML + * @param type the type of log entry, one of {@link #USER_CREATION_LOG}, + * {@link #DELETION_LOG}, {@link #BLOCK_LOG}, etc. + * @param action the type of action that was performed e.g. "delete", + * "unblock", "overwrite", etc. + * @param target the target of the action + * @param details the details of the action (e.g. the new title of + * the page after a move was performed). + * @since 0.08 + */ + protected LogEntry(long id, OffsetDateTime timestamp, String user, String comment, + String parsedcomment, String type, String action, String target, Map details) + { + super(id, timestamp, user, target, comment, parsedcomment); + this.type = Objects.requireNonNull(type); + this.action = action; + this.details = details; } /** - * Gets the target of the action represented by this log entry. WARNING: - * returns null if the content was RevisionDeleted and one does not - * have access to the content. - * @return the target of this log entry + * Gets the type of log that this entry is in. + * @return one of {@link Wiki#DELETION_LOG}, {@link Wiki#BLOCK_LOG}, etc. * @since 0.08 */ - public String getTarget() + public String getType() { - return target; + return type; } /** - * Gets the timestamp of this log entry. - * @return the timestamp of this log entry + * Gets a string description of the action performed, for example + * "delete", "protect", "overwrite", ... WARNING: returns null if the + * action was RevisionDeleted. + * @return the type of action performed * @since 0.08 */ - public OffsetDateTime getTimestamp() + public String getAction() { - return timestamp; + return action; } /** @@ -6309,13 +7124,14 @@ public OffsetDateTime getTimestamp() * USER_RENAME_LOG * The new username * BLOCK_LOG - * new Object[] { boolean anononly, boolean nocreate, boolean + * new Object[] { boolean anononly, boolean nocreate, boolean * noautoblock, boolean noemail, boolean nousertalk, String duration } * USER_RIGHTS_LOG - * The new user rights (String[]) + * The old ("oldgroups") and new ("newgroups") user rights, + * comma-separated * PROTECTION_LOG * if action == "protect" or "modify" return the protection level - * (int, -2 if unrecognized) if action == "move_prot" return + * (int, -2 if unrecognized) if action == "move_prot" return * the old title, else null * Others or RevisionDeleted * null @@ -6329,7 +7145,7 @@ public OffsetDateTime getTimestamp() * @return the details of the log entry * @since 0.08 */ - public Object getDetails() + public Map getDetails() { return details; } @@ -6342,44 +7158,21 @@ public Object getDetails() @Override public String toString() { - StringBuilder s = new StringBuilder("LogEntry[logid="); - s.append(logid); + StringBuilder s = new StringBuilder(super.toString()); + s.deleteCharAt(s.length() - 1); s.append(",type="); s.append(type); s.append(",action="); - s.append(action == null ? "[hidden]" : action); - s.append(",user="); - s.append(user == null ? "[hidden]" : user.getUsername()); - s.append(",timestamp="); - s.append(timestamp.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - s.append(",target="); - s.append(target == null ? "[hidden]" : target); - s.append(",reason=\""); - s.append(reason == null ? "[hidden]" : reason); - s.append("\",details="); - if (details instanceof Object[]) - s.append(Arrays.asList((Object[])details)); // crude formatting hack - else - s.append(details); + s.append(Objects.toString(action, "[DELETED]")); + s.append(",details="); + s.append(details); s.append("]"); return s.toString(); } /** - * Compares this log entry to another one based on the recentness - * of their timestamps. - * @param other the log entry to compare - * @return whether this object is equal to - * @since 0.18 - */ - @Override - public int compareTo(Wiki.LogEntry other) - { - return timestamp.compareTo(other.timestamp); - } - - /** - * Determines whether two LogEntries refer to the same event. + * Determines whether two LogEntries are equal based on the underlying + * {@linkplain Event#equals(Object) Event}, type and action. * @param other some object to compare to * @return (see above) * @since 0.33 @@ -6387,12 +7180,28 @@ public int compareTo(Wiki.LogEntry other) @Override public boolean equals(Object other) { + if (!super.equals(other)) + return false; if (!(other instanceof LogEntry)) return false; LogEntry le = (LogEntry)other; - return type.equals(le.type) && action.equals(le.action) && - user.equals(le.user) && target.equals(le.target) && - reason.equals(le.reason) && timestamp.equals(le.timestamp); + return Objects.equals(type, le.type) + && Objects.equals(action, le.action); + } + + /** + * Computes a hashcode for this LogEntry based on the underlying + * {@linkplain Event#hashCode() Event}, type and action. + * @return (see above) + * @since 0.35 + */ + @Override + public int hashCode() + { + int hc = super.hashCode(); + hc = 127 * hc + type.hashCode(); + hc = 127 * hc + Objects.hashCode(action); + return hc; } } @@ -6400,46 +7209,39 @@ public boolean equals(Object other) * Represents a contribution and/or a revision to a page. * @since 0.17 */ - public class Revision implements Comparable + public class Revision extends Event { - private boolean minor, bot, rvnew; - private String summary; - private long revid, rcid = -1; + private final boolean minor, bot, rvnew; + private final String sha1; + private long rcid = -1; private long previous = 0, next = 0; - private OffsetDateTime timestamp; - private String user; - private String title; - private String rollbacktoken = null; - private int size = 0; - private int sizediff = 0; - private boolean summaryDeleted = false, userDeleted = false, contentDeleted = false; + private int size = 0, sizediff = 0; private boolean pageDeleted = false; /** * Constructs a new Revision object. * @param revid the id of the revision (this is a long since - * {{NUMBEROFEDITS}} on en.wikipedia.org is now (January 2012) ~25% - * of Integer.MAX_VALUE + * {{NUMBEROFEDITS}} on en.wikipedia.org is now (January 2018) ~38% + * of {@code Integer.MAX_VALUE} * @param timestamp when this revision was made + * @param user the user making this revision (may be anonymous) + * @param comment the edit summary + * @param parsedcomment the edit summary, parsed into HTML * @param title the concerned article - * @param summary the edit summary - * @param user the user making this revision (may be anonymous, if not - * use User.getUsername()) + * @param sha1 the SHA-1 hash of the revision * @param minor whether this was a minor edit * @param bot whether this was a bot edit * @param rvnew whether this revision created a new page * @param size the size of the revision * @since 0.17 */ - public Revision(long revid, OffsetDateTime timestamp, String title, String summary, String user, - boolean minor, boolean bot, boolean rvnew, int size) + public Revision(long revid, OffsetDateTime timestamp, String user, String comment, + String parsedcomment, String title, String sha1, boolean minor, boolean bot, + boolean rvnew, int size) { - this.revid = revid; - this.timestamp = timestamp; - this.summary = summary; + super(revid, timestamp, user, Objects.requireNonNull(title), comment, parsedcomment); + this.sha1 = sha1; this.minor = minor; - this.user = user; - this.title = title; this.bot = bot; this.rvnew = rvnew; this.size = size; @@ -6447,7 +7249,7 @@ public Revision(long revid, OffsetDateTime timestamp, String title, String summa /** * Fetches the contents of this revision. - * @return the contents of the appropriate article at timestamp + * @return the contents of the appropriate article at timestamp * @throws IOException if a network error occurs * @throws IllegalArgumentException if page == Special:Log/xxx. * @since 0.17 @@ -6455,176 +7257,125 @@ public Revision(long revid, OffsetDateTime timestamp, String title, String summa public String getText() throws IOException { // logs have no content - if (revid < 1L) + if (getID() < 1L) throw new IllegalArgumentException("Log entries have no valid content!"); // TODO: returning a 404 here when revision content has been deleted // is not a good idea. if (pageDeleted) // FIXME: broken if a page is live, but has deleted revisions { - String url = query + "prop=deletedrevisions&drvprop=content&revids=" + revid; - String temp = fetch(url, "Revision.getText"); + Map getparams = new HashMap<>(); + getparams.put("action", "query"); + getparams.put("prop", "deletedrevisions"); + getparams.put("drvprop", "content"); + getparams.put("revids", String.valueOf(getID())); + String temp = makeApiCall(getparams, null, "Revision.getText"); + detectUncheckedErrors(temp, null, null); int a = temp.indexOf("", a) + 1; + a = temp.indexOf('>', a) + 1; int b = temp.indexOf("", a); // tag not present if revision has no content - log(Level.INFO, "Revision.getText", "Successfully retrieved text of revision " + revid); + log(Level.INFO, "Revision.getText", "Successfully retrieved text of revision " + getID()); return (b < 0) ? "" : temp.substring(a, b); } else - { - String url = base + encode(title, true) + "&oldid=" + revid + "&action=raw"; - String temp = fetch(url, "Revision.getText"); - log(Level.INFO, "Revision.getText", "Successfully retrieved text of revision " + revid); - return temp; - } + return Wiki.this.getText(null, new long[] { getID() }, -1).get(0); } /** * Gets the rendered text of this revision. * @return the rendered contents of the appropriate article at - * timestamp + * timestamp * @throws IOException if a network error occurs * @throws IllegalArgumentException if page == Special:Log/xxx. * @since 0.17 */ public String getRenderedText() throws IOException { - // logs have no content - if (revid < 1L) + if (getID() < 1L) throw new IllegalArgumentException("Log entries have no valid content!"); - - String temp; - if (pageDeleted) - { - String url = query + "prop=deletedrevisions&drvprop=content&drvparse=1&revids=" + revid; - temp = fetch(url, "Revision.getRenderedText"); - // TODO - } - else - { - String url = base + "&action=render&oldid=" + revid; - temp = fetch(url, "Revision.getRenderedText"); - } - log(Level.INFO, "Revision.getRenderedText", "Successfully retrieved rendered text of revision " + revid); - return decode(temp); + Map content = new HashMap<>(); + content.put("revision", this); + return Wiki.this.parse(content, -1, false); } /** - * Returns true if the revision content is RevisionDeleted. + * Returns the SHA-1 hash (base 16, lower case) of the content of this + * revision, or {@code null} if the revision content is RevisionDeleted + * and we cannot access it. + * + *

Warnings: + *

    + *
  • Not available through {@link #watchlist(RequestHelper)} or {@link + * #contribs(List, String, RequestHelper)}. + *
+ * * @return (see above) - * @since 0.31 + * @since 0.35 */ - public boolean isContentDeleted() + public String getSha1() { - return contentDeleted; + return sha1; } /** - * Returns a HTML rendered diff table; see the table at the example. - * Returns null for page creations, moves, protections and similar - * dummy edits ( - * example). - * + * Returns a HTML rendered diff table of this revision to other. + * See {@link #diff(Map, Map)} for full documentation. + * * @param other another revision on the same page. * @return the difference between this and the other revision * @throws IOException if a network error occurs + * @throws SecurityException if this or the other revision is + * RevisionDeleted and the user lacks the necessary privileges * @since 0.21 */ public String diff(Revision other) throws IOException { - return diff(other.revid, ""); + return Wiki.this.diff(Map.of("revision", this), Map.of("revision", other)); } /** - * Returns a HTML rendered diff table between this revision and the - * given text. Useful for emulating the "show changes" functionality. - * See the table at the text. Useful for emulating the "show changes" + * functionality. See the table at the example. + * * @param text some wikitext * @return the difference between this and the the text provided * @throws IOException if a network error occurs + * @throws SecurityException if this or the other revision is + * RevisionDeleted and the user lacks the necessary privileges * @since 0.21 */ public String diff(String text) throws IOException { - return diff(0L, text); + return Wiki.this.diff(Map.of("revision", this), Map.of("text", text)); } /** - * Returns a HTML rendered diff table; see the table at the example. - * Returns null for page creations, moves, protections and - * similar dummy edits (example) - * and pairs of revisions where there is no difference. - * - * @param oldid the oldid of a revision on the same page. {@link Wiki#NEXT_REVISION}, - * {@link Wiki#PREVIOUS_REVISION} and {@link Wiki#CURRENT_REVISION} can - * be used here for obvious effect. + * Returns a HTML rendered diff table from this revision to the given + * oldid. See {@link #diff(Map, Map)} for full documentation. + * + * @param oldid the oldid of a revision on the same page. {@link + * Wiki#NEXT_REVISION}, {@link Wiki#PREVIOUS_REVISION} and {@link + * Wiki#CURRENT_REVISION} can be used here for obvious effect. * @return the difference between this and the other revision * @throws IOException if a network error occurs + * @throws SecurityException if this or the other revision is + * RevisionDeleted and the user lacks the necessary privileges * @since 0.26 */ public String diff(long oldid) throws IOException { - return diff(oldid, ""); - } - - /** - * Fetches a HTML rendered diff table; see the table at the example. - * Returns null for page creations, moves, protections and similar - * dummy edits ( - * example) and pairs of revisions where there is no difference. - * - * @param oldid the id of another revision; (exclusive) or - * @param text some wikitext to compare against - * @return a difference between oldid or text or null if there is no - * diff. - * @throws IOException if a network error occurs - * @since 0.21 - */ - protected String diff(long oldid, String text) throws IOException - { - StringBuilder temp = new StringBuilder(); - if (oldid == NEXT_REVISION) - temp.append("&torelative=next"); - else if (oldid == CURRENT_REVISION) - temp.append("&torelative=cur"); - else if (oldid == PREVIOUS_REVISION) - temp.append("&torelative=prev"); - else if (oldid == 0L) - { - temp.append("&totext="); - temp.append(text); - } - else - { - temp.append("&torev="); - temp.append(oldid); - } - - String line = post(apiUrl + "action=compare&fromrev=" + getRevid(), temp.toString(), "Revision.diff"); - - // strip extraneous information - if (line.contains("")) - { - int a = line.indexOf("", a) + 1; - int b = line.indexOf("", a); - return decode(line.substring(a, b)); - } - else - // tag has no content if there is no diff or the two - // revisions are identical. In particular, the API does not - // distinguish between: - // https://en.wikipedia.org/w/index.php?title=Source_Filmmaker&diff=804972897&oldid=803731343 (no difference) - // https://en.wikipedia.org/w/index.php?title=Dayo_Israel&oldid=738178354&diff=prev (dummy edit) - return null; + Map from = new HashMap<>(); + from.put("revision", this); + Map to = new HashMap<>(); + to.put("revid", oldid); + return Wiki.this.diff(from, to); } /** - * Determines whether this Revision is equal to another object. + * Determines whether this Revision is equal to another based on the + * underlying {@linkplain Event#equals(Object) Event}. * @param o an object * @return whether o is equal to this object * @since 0.17 @@ -6632,20 +7383,29 @@ else if (oldid == 0L) @Override public boolean equals(Object o) { + // Note to self: don't use SHA-1 until all API calls provide it. + if (!super.equals(o)) + return false; if (!(o instanceof Revision)) return false; - return revid == ((Revision)o).revid; + // Revision rev = (Revision)o; + // return Objects.equals(sha1, rev.sha1); + return true; } /** - * Returns a hash code of this revision. + * Returns a hash code of this revision based on the underlying + * {@linkplain Event#hashCode() Event}. * @return a hash code * @since 0.17 */ @Override public int hashCode() { - return (int)revid * 2 - Wiki.this.hashCode(); + // Note to self: don't use SHA-1 until all API calls provide it. + int hc = super.hashCode(); + hc = 127 * hc; + return hc; } /** @@ -6671,12 +7431,15 @@ public boolean isBot() } /** - * Determines whether this revision created a new page.
- * WARNING: Will return false for all revisions prior to 2007 - * (I think?) -- this is a MediaWiki problem.
- * WARNING: Returning true does not imply this is the bottommost - * revision on the page due to histmerges.
- * WARNING: Not accessible through getPageHistory() -- a MW problem. + * Determines whether this revision created a new page. + * + *

Warnings: + *

    + *
  • Returning {@code true} does not imply this is the bottommost + * revision on the page due to histmerges. + *
  • Not available through {@link #getPageHistory(String, Wiki.RequestHelper)} + *
+ * * @return (see above) * @since 0.27 */ @@ -6686,51 +7449,8 @@ public boolean isNew() } /** - * Returns the edit summary for this revision, or null if the summary - * was RevisionDeleted and you lack the necessary privileges. - * @return the edit summary - * @since 0.17 - */ - public String getSummary() - { - return summary; - } - - /** - * Returns true if the edit summary is RevisionDeleted. - * @return (see above) - * @since 0.30 - */ - public boolean isSummaryDeleted() - { - return summaryDeleted; - } - - /** - * Returns the user or anon who created this revision. You should - * pass this (if not an IP) to getUser(String) to obtain a - * User object. Returns null if the user was RevisionDeleted and you - * lack the necessary privileges. - * @return the user or anon - * @since 0.17 - */ - public String getUser() - { - return user; - } - - /** - * Returns true if the user is RevisionDeleted. - * @return (see above) - * @since 0.30 - */ - public boolean isUserDeleted() - { - return userDeleted; - } - - /** - * Returns true if this revision is deleted (different from revdeleted). + * Returns {@code true} if this revision is deleted (not the same as + * RevisionDeleted). * @return (see above) * @since 0.31 */ @@ -6739,37 +7459,6 @@ public boolean isPageDeleted() return pageDeleted; } - /** - * Returns the page to which this revision was made. - * @return the page - * @since 0.17 - */ - public String getPage() - { - return title; - } - - /** - * Returns the oldid of this revision. Don't confuse this with - * rcid - * @return the oldid (long) - * @since 0.17 - */ - public long getRevid() - { - return revid; - } - - /** - * Gets the time that this revision was made. - * @return the timestamp - * @since 0.17 - */ - public OffsetDateTime getTimestamp() - { - return timestamp; - } - /** * Gets the size of this revision in bytes. * @return see above @@ -6781,7 +7470,8 @@ public int getSize() } /** - * Returns the change in page size caused by this revision. + * Returns the change in page size caused by this revision. Not available + * through getPageHistory or getDeletedHistory. * @return see above * @since 0.28 */ @@ -6798,22 +7488,8 @@ public int getSizeDiff() @Override public String toString() { - StringBuilder sb = new StringBuilder("Revision[oldid="); - sb.append(revid); - sb.append(",page=\""); - sb.append(title); - sb.append("\",user="); - sb.append(user == null ? "[hidden]" : user); - sb.append(",userdeleted="); - sb.append(userDeleted); - sb.append(",timestamp="); - sb.append(timestamp.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); - sb.append(",summary=\""); - sb.append(summary == null ? "[hidden]" : summary); - sb.append("\",summarydeleted="); - sb.append(summaryDeleted); - sb.append(",contentDeleted="); - sb.append(contentDeleted); + StringBuilder sb = new StringBuilder(super.toString()); + sb.deleteCharAt(sb.length() - 1); sb.append(",minor="); sb.append(minor); sb.append(",bot="); @@ -6826,25 +7502,10 @@ public String toString() sb.append(previous); sb.append(",next="); sb.append(next); - sb.append(",rollbacktoken="); - sb.append(rollbacktoken == null ? "null" : rollbacktoken); sb.append("]"); return sb.toString(); } - /** - * Compares this revision to another revision based on the recentness - * of their timestamps. - * @param other the revision to compare - * @return whether this object is equal to - * @since 0.18 - */ - @Override - public int compareTo(Wiki.Revision other) - { - return timestamp.compareTo(timestamp); - } - /** * Gets the previous revision. * @return the previous revision, or null if this is the first revision @@ -6892,35 +7553,25 @@ public long getRcid() } /** - * Sets a rollback token for this revision. - * @param token a rollback token - * @since 0.24 - */ - public void setRollbackToken(String token) - { - rollbacktoken = token; - } - - /** - * Gets the rollback token for this revision. Can be null, and often - * for good reasons: cannot rollback or not top revision. - * @return the rollback token - * @since 0.24 + * Gets a permanent URL to the human-readable version of this Revision. + * This URL uses index.php, not Special:Permanentlink for ease of adding + * other parameters. + * @return (see above) + * @since 0.35 */ - public String getRollbackToken() + public String permanentUrl() { - return rollbacktoken; + return getIndexPhpUrl() + "?oldid=" + getID(); } /** - * Reverts this revision using the rollback method. See - * {@link Wiki#rollback(org.wikipedia.Wiki.Revision)}. - * + * Reverts this revision using the rollback method. + * * @throws IOException if a network error occurs - * @throws CredentialNotFoundException if not logged in or user is not - * an admin + * @throws SecurityException if the user lacks the privileges to rollback * @throws CredentialExpiredException if cookies have expired * @throws AccountLockedException if the user is blocked + * @see Wiki#rollback(org.wikipedia.Wiki.Revision) * @since 0.19 */ public void rollback() throws IOException, LoginException @@ -6929,174 +7580,588 @@ public void rollback() throws IOException, LoginException } /** - * Reverts this revision using the rollback method. See - * {@link Wiki#rollback(org.wikipedia.Wiki.Revision)}. - * + * Reverts this revision using the rollback method. + * * @param bot mark this and the reverted revision(s) as bot edits * @param reason (optional) a custom reason * @throws IOException if a network error occurs - * @throws CredentialNotFoundException if not logged in or user is not - * an admin + * @throws SecurityException if the user lacks the privileges to rollback * @throws CredentialExpiredException if cookies have expired * @throws AccountLockedException if the user is blocked + * @see Wiki#rollback(org.wikipedia.Wiki.Revision) * @since 0.19 */ - public void rollback(boolean bot, String reason) throws IOException, LoginException + public void rollback(boolean bot, String reason) throws IOException, LoginException + { + Wiki.this.rollback(this, bot, reason); + } + } + + /** + * Vehicle for stuffing standard optional parameters into Wiki queries. + * {@code RequestHelper} objects are reusable. The following example + * fetches articles from the back of the new pages queue on the + * English Wikipedia. + * + * {@code 
+     *  Wiki.RequestHelper rh = enWiki.new RequestHelper()
+     *      .inNamespaces(Wiki.MAIN_NAMESPACE)
+     *      .reverse();
+     *  List newpages = enWiki.newPages(rh);
+     *
} + * + * @since 0.36 + */ + public class RequestHelper + { + private String title; + private String byuser; + private OffsetDateTime earliest, latest; + private int[] localns = new int[0]; + private boolean reverse = false; + private String notbyuser; + private String tag; + private Map options; + private String requestType; + private int limit = -1; + + /** + * Creates a new RequestHelper. + */ + public RequestHelper() + { + } + + /** + * Limits query results to Events occuring on the given title. If a + * query mandates a title parameter (e.g. {@link #getPageHistory(String, + * RequestHelper)}, don't use this. Use the parameter in the query + * method instead. + * @param title a page title + * @return this RequestHelper + */ + public RequestHelper byTitle(String title) + { + this.title = (title == null) ? null : normalize(title); + return this; + } + + /** + * Limits query results to Events triggered by the given user. If a query + * mandates a user parameter (e.g. {@link #contribs(List, String, RequestHelper)}, + * don't use this. Use the parameter in the query method instead. + * @param byuser some username or IP address + * @return this RequestHelper + */ + public RequestHelper byUser(String byuser) + { + this.byuser = (byuser == null) ? null : normalize(byuser); + return this; + } + + /** + * Limit results to be within this date range. + * @param earliest the lower (earliest) date bound, use {@code null} to + * not set one + * @param latest the higher (latest) date bound, use {@code null} to + * not set one + * @throws IllegalArgumentException if {@code earliest.isAfter(latest)} + * @return this RequestHelper + */ + public RequestHelper withinDateRange(OffsetDateTime earliest, OffsetDateTime latest) + { + if (earliest != null && latest != null && earliest.isAfter(latest)) + throw new IllegalArgumentException("Earliest date must be before latest date!"); + this.earliest = earliest; + this.latest = latest; + return this; + } + + /** + * Limits query results to the given namespaces. + * @param ns a list of namespaces + * @return this RequestHelper + */ + public RequestHelper inNamespaces(int... ns) + { + localns = ns; + return this; + } + + /** + * Should we perform this query in reverse order (earliest first). + * @param reverse whether to reverse this query + * @return this RequestHelper + */ + public RequestHelper reverse(boolean reverse) + { + this.reverse = reverse; + return this; + } + + /** + * Limits query results to {@link Event Events} that have been tagged + * with the given tag. + * @param tag a change tag + * @return this RequestHelper + */ + public RequestHelper taggedWith(String tag) + { + this.tag = tag; + return this; + } + + /** + * Limits query results to Events NOT triggered by the given user. + * @param notbyuser some username or IP address to exclude + * @return this RequestHelper + */ + public RequestHelper notByUser(String notbyuser) + { + this.notbyuser = (notbyuser == null) ? null : normalize(notbyuser); + return this; + } + + /** + * Return no more than the given number of results. Overrides {@linkplain + * #setQueryLimit(int) global limits}. Supply a negative integer to use + * global limits. + * @param limit a positive integer + * @return this RequestHelper + */ + public RequestHelper limitedTo(int limit) + { + this.limit = limit; + return this; + } + + /** + * Filters a set of returned results using the given options. Please + * check calling method documentation for supported options. + * + *

+ * When filtering revisions, available keys may include "minor", "top", + * "new", "bot", "anon", "redirect", "patrolled" and "unread" for + * vanilla MediaWiki. Extensions may define their own. For instance, + * {@code rcoptions = { minor = true, anon = false, patrolled = false}} + * returns all minor edits from logged in users that aren't patrolled. + * Setting "patrolled" limits results to no older than retention in + * the recentchanges + * table. + * + * @param options the options to filter by + * @return this RequestHelper + */ + public RequestHelper filterBy(Map options) + { + this.options = options; + return this; + } + + /** + * Sets the prefix of API request parameters (the XX in XXlimit, XXdir, + * XXnamespace and so forth). Internal use only. + * @param prefix the prefix to use (must not be null) + */ + protected void setRequestType(String prefix) + { + requestType = Objects.requireNonNull(prefix); + } + + /** + * Returns a HTTP request parameter containing the title to get + * events for, or an empty map if not wanted. + * @return (see above) + */ + protected Map addTitleParameter() + { + if (title != null) + return Map.of(requestType + "title", title); + return Collections.emptyMap(); + } + + /** + * Returns a HTTP request parameter containing the user to filter + * returned events by, or an empty map if not wanted. + * @return (see above) + */ + protected Map addUserParameter() + { + if (byuser != null) + return Map.of(requestType + "user", byuser); + return Collections.emptyMap(); + } + + /** + * Returns a HTTP request parameter containing the dates to start + * and end enumeration, or an empty map if not wanted. + * @return (see above) + */ + protected Map addDateRangeParameters() + { + // https://phabricator.wikimedia.org/T16449 + Map temp = new HashMap<>(); + OffsetDateTime odt = reverse ? earliest : latest; + if (odt != null) + temp.put(requestType + "start", + // https://www.mediawiki.org/wiki/Timestamp + // https://github.com/MER-C/wiki-java/issues/170 + odt.withOffsetSameInstant(ZoneOffset.UTC) + .truncatedTo(ChronoUnit.MICROS) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + odt = reverse ? latest : earliest; + if (odt != null) + temp.put(requestType + "end", + // https://www.mediawiki.org/wiki/Timestamp + // https://github.com/MER-C/wiki-java/issues/170 + odt.withOffsetSameInstant(ZoneOffset.UTC) + .truncatedTo(ChronoUnit.MICROS) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)); + return temp; + } + + /** + * Returns a HTTP request parameter containing the namespaces to limit + * this query to, or an empty map if not wanted. + * @return (see above) + */ + protected Map addNamespaceParameter() + { + if (localns.length != 0) + return Map.of(requestType + "namespace", constructNamespaceString(localns)); + return Collections.emptyMap(); + } + + /** + * Returns a HTTP request parameter instructing the API to reverse the + * query, or an empty map if not wanted. + * @return (see above) + */ + protected Map addReverseParameter() + { + return Map.of(requestType + "dir", reverse ? "newer" : "older"); + } + + /** + * Returns a HTTP request parameter containing the tag to limit + * returned events to, or an empty map if not wanted. + * @return (see above) + */ + protected Map addTagParameter() + { + if (tag != null) + return Map.of(requestType + "tag", tag); + return Collections.emptyMap(); + } + + /** + * Returns a HTTP request parameter containing the user to exclude + * when returning events, or an empty map if not wanted. + * @return (see above) + */ + protected Map addExcludeUserParameter() + { + if (notbyuser != null) + return Map.of(requestType + "excludeuser", notbyuser); + return Collections.emptyMap(); + } + + /** + * Returns HTTP request parameter(s) containing flags to filter returned + * revisions by, or an empty map if not wanted. + * @return (see above) + */ + protected Map addShowParameter() + { + Map temp = new HashMap<>(); + if (options != null && !options.isEmpty()) + { + // deal with MW API annoyance for action=query&list=watchlist - see watchlist(rh) + Boolean top = null; + if (requestType.equals("wl")) + { + top = options.remove("top"); + if (Boolean.TRUE.equals(top)) + temp.put("wlallrev", "1"); + } + + StringBuilder sb = new StringBuilder(); + options.forEach((key, value) -> + { + if (Boolean.FALSE.equals(value)) + sb.append('!'); + sb.append(key); + sb.append("|"); + }); + temp.put(requestType + "show", sb.substring(0, sb.length() - 1)); + + if (top != null) // put it back + options.put("top", top); + } + return temp; + } + + /** + * Returns the number of results the query should be limited to. If not + * present, use {@linkplain #setQueryLimit(int) global limits}. + * @return (see above) + */ + public int limit() { - Wiki.this.rollback(this, bot, reason); + return limit; } } // INTERNALS + /** + * Performs a vectorized action=query&prop=X type API query + * over titles. + * @param queryPrefix the request type prefix (e.g. "pl" for prop=links) + * @param getparams a bunch of parameters to send via HTTP GET + * @param titles a list of titles + * @param caller the name of the calling method + * @param limit fetch no more than this many results + * @param parser a BiConsumer that parses the XML returned by the MediaWiki + * API into things we want, dumping them into the given List + * @return a list of results, where each element corresponds to the element + * at the same index in the input title list + * @since 0.36 + * @throws IOException if a network error occurs + */ + protected List> makeVectorizedQuery(String queryPrefix, Map getparams, + List titles, String caller, int limit, BiConsumer> parser) throws IOException + { + // copy because normalization and redirect resolvers overwrite + List titles2 = new ArrayList<>(titles); + List>> stuff = new ArrayList<>(); + Map postparams = new HashMap<>(); + for (String temp : constructTitleString(titles2)) + { + postparams.put("titles", temp); + stuff.addAll(makeListQuery(queryPrefix, getparams, postparams, caller, -1, (line, results) -> + { + // Split the result into individual listings for each article. + String[] x = line.split(" list = new ArrayList<>(); + parser.accept(x[i], list); + + Map> intermediate = new HashMap<>(); + intermediate.put(parsedtitle, list); + results.add(intermediate); + } + })); + } + + // prepare the return list + List> ret = Stream.generate(() -> new ArrayList()) + .limit(titles2.size()) + .collect(Collectors.toCollection(ArrayList::new)); + // then retrieve the results from the intermediate list of maps, + // ensuring results correspond to inputs + stuff.forEach(map -> + { + String parsedtitle = map.keySet().iterator().next(); + List templates = map.get(parsedtitle); + for (int i = 0; i < titles2.size(); i++) + if (titles2.get(i).equals(parsedtitle)) + ret.get(i).addAll(templates); + }); + return ret; + } + /** * Fetches list-type results from the MediaWiki API. - * - * @param a class describing the parsed API results (e.g. String, + * + * @param a class describing the parsed API results (e.g. String, * LogEntry, Revision) * @param queryPrefix the request type prefix (e.g. "pl" for prop=links) - * @param url the query URL, without the limit and XXcontinue parameters + * @param getparams a bunch of parameters to send via HTTP GET + * @param postparams if not null, send these parameters via POST (see + * {@link #makeApiCall(Map, Map, String) }). * @param caller the name of the calling method + * @param limit fetch no more than this many results * @param parser a BiConsumer that parses the XML returned by the MediaWiki * API into things we want, dumping them into the given List * @return the query results * @throws IOException if a network error occurs + * @throws SecurityException if we don't have the credentials to perform a + * privileged action (mostly avoidable) * @since 0.34 */ - protected List queryAPIResult(String queryPrefix, StringBuilder url, String caller, - BiConsumer> parser) throws IOException + protected List makeListQuery(String queryPrefix, Map getparams, + Map postparams, String caller, int limit, BiConsumer> parser) throws IOException { + if (limit < 0) + limit = querylimit; + getparams.put("action", "query"); List results = new ArrayList<>(1333); - StringBuilder xxcontinue = new StringBuilder(); - url.append("&"); - url.append(queryPrefix); - url.append("limit="); + String limitstring = queryPrefix + "limit"; do { - int limit = Math.min(querylimit - results.size(), max); - String tempurl = url.toString() + limit; - String line = fetch(tempurl + xxcontinue.toString(), caller); - xxcontinue.setLength(0); - + getparams.put(limitstring, String.valueOf(Math.min(limit - results.size(), max))); + String line = makeApiCall(getparams, postparams, caller); + detectUncheckedErrors(line, null, null); + getparams.keySet().removeIf(param -> param.endsWith("continue")); + // Continuation parameter has form: // if (line.contains("", a); - String[] temp = line.substring(a, b).split("\""); - xxcontinue.append("&"); - xxcontinue.append(temp[0]); - xxcontinue.append(encode(temp[1], false)); - xxcontinue.append(temp[2].replace(" ", "&")); - xxcontinue.append(encode(temp[3], false)); + int a = line.indexOf("", a); + String cont = line.substring(a, b); + for (String contpair : cont.split("\" ")) + { + contpair = " " + contpair.trim(); + String contattr = contpair.substring(0, contpair.indexOf("=\"")); + getparams.put(contattr.trim(), parseAttribute(cont, contattr, 0)); + } } - + parser.accept(line, results); } - while (xxcontinue.length() != 0 && results.size() < querylimit); + while (getparams.containsKey("continue") && results.size() < limit); return results; } - - // miscellany + // miscellany + + /** + * Convenience method for checking user permissions. + * @param right a user rights + * @param morerights additional user rights + * @throws SecurityException if the permission check fails + * @since 0.37 + */ + protected void checkPermissions(String action, String right, String... morerights) + { + if (user == null) + throw new SecurityException("Cannot " + action + ": not logged in."); + if (!user.isAllowedTo(right, morerights)) + throw new SecurityException("Cannot " + action + ": permission denied."); + } + /** - * A generic URL content fetcher. This is only useful for GET requests, - * which is almost everything that doesn't modify the wiki. Might be - * useful for subclasses. + * Constructs, sends and handles calls to the MediaWiki API. This is a + * low-level method for making your own, custom API calls. + * + *

+ * If postparams is not {@code null} or empty, the request is + * sent using HTTP GET, otherwise it is sent using HTTP POST. A + * {@code byte[]} value in postparams causes the request to be + * sent as a multipart POST. Anything else is converted to String via the + * following means: + * + *

    + *
  • String[] -- {@code String.join("|", arr)} + *
  • StringBuilder -- {@code sb.toString()} + *
  • Number -- {@code num.toString()} + *
  • OffsetDateTime -- {@code date.format(DateTimeFormatter.ISO_OFFSET_DATE_TIME)} + *
  • {@code Collection} -- {@code coll.stream() + * .map(item -> convertToString(item)) // using the above rules + * .collect(Collectors.joining("|"))} + *
+ * + *

+ * All supplied Strings and objects converted to String are automatically + * URLEncoded in UTF-8 if this is a normal POST request. * + *

* Here we also check the database lag and wait if it exceeds * maxlag, see * here for how this works. * - * @param url the url to fetch + * @param getparams append these parameters to the urlbase + * @param postparams if null, send the request using POST otherwise use GET * @param caller the caller of this method - * @return the content of the fetched URL + * @return the server response * @throws IOException if a network error occurs + * @throws SecurityException if we don't have the credentials to perform a + * privileged action (mostly avoidable) * @throws AssertionError if assert=user|bot fails + * @see Multipart/form-data * @since 0.18 */ - protected String fetch(String url, String caller) throws IOException + public String makeApiCall(Map getparams, Map postparams, String caller) throws IOException { - String temp = null; - int tries = maxtries; - do + // build the URL + StringBuilder urlbuilder = new StringBuilder(apiUrl + "?"); + getparams.putAll(defaultApiParams); + for (Map.Entry entry : getparams.entrySet()) { - logurl(url, caller); - tries--; - try + urlbuilder.append('&'); + urlbuilder.append(entry.getKey()); + urlbuilder.append('='); + urlbuilder.append(URLEncoder.encode(entry.getValue(), StandardCharsets.UTF_8)); + } + String url = urlbuilder.toString(); + + // POST stuff + boolean isPOST = (postparams != null && !postparams.isEmpty()); + StringBuilder stringPostBody = new StringBuilder(); + boolean multipart = false; + ArrayList multipartPostBody = new ArrayList<>(); + String boundary = "----------NEXT PART----------"; + if (isPOST) + { + // determine whether this is a multipart post and convert any values + // to String if necessary + for (Map.Entry entry : postparams.entrySet()) { - // connect - URLConnection connection = makeConnection(url); - connection.setConnectTimeout(CONNECTION_CONNECT_TIMEOUT_MSEC); - connection.setReadTimeout(CONNECTION_READ_TIMEOUT_MSEC); - setCookies(connection); - connection.connect(); - grabCookies(connection); - - // check lag and retry - if (checkLag(connection)) - return fetch(url, caller); - - // get the text - String line; - StringBuilder text = new StringBuilder(100000); - try (BufferedReader in = new BufferedReader(new InputStreamReader( - zipped ? new GZIPInputStream(connection.getInputStream()) : connection.getInputStream(), "UTF-8"))) + Object value = entry.getValue(); + if (value instanceof byte[]) + multipart = true; + else + entry.setValue(convertToString(value)); + } + + // now we know how we're sending it, construct the post body + if (multipart) + { + byte[] nextpart = ("--" + boundary + "\r\n\"Content-Disposition: form-data; name=\\\"\"") + .getBytes(StandardCharsets.UTF_8); + for (Map.Entry entry : postparams.entrySet()) { - while ((line = in.readLine()) != null) + multipartPostBody.add(nextpart); + Object value = entry.getValue(); + multipartPostBody.add((entry.getKey() + "\"\r\n").getBytes(StandardCharsets.UTF_8)); + if (value instanceof String) + multipartPostBody.add(("Content-Type: text/plain; charset=UTF-8\r\n\r\n" + (String)value + "\r\n") + .getBytes(StandardCharsets.UTF_8)); + else if (value instanceof byte[]) { - text.append(line); - text.append("\n"); + multipartPostBody.add("Content-Type: application/octet-stream\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + multipartPostBody.add((byte[])value); + multipartPostBody.add("\r\n".getBytes(StandardCharsets.UTF_8)); } } - temp = text.toString(); + multipartPostBody.add((boundary + "--\r\n").getBytes(StandardCharsets.UTF_8)); } - catch (IOException ex) + else { - if (tries == 0) - throw ex; - try - { - Thread.sleep(10000); - } - catch (InterruptedException ignored) + // automatically encode Strings sent via normal POST + for (Map.Entry entry : postparams.entrySet()) { + stringPostBody.append('&'); + stringPostBody.append(entry.getKey()); + stringPostBody.append('='); + stringPostBody.append(URLEncoder.encode(entry.getValue().toString(), StandardCharsets.UTF_8)); } } } - while (temp == null); - if (temp.contains(" hr = client.send(connection.build(), HttpResponse.BodyHandlers.ofInputStream()); + boolean zipped_ = hr.headers().firstValue("Content-Encoding").orElse("").equals("gzip"); + if (checkLag(hr)) { - out.write(text); + tries++; + throw new HttpRetryException("Database lagged.", 503); } - // check lag and retry - if (checkLag(connection)) - return post(url, text, caller); - grabCookies(connection); - StringBuilder buffer = new StringBuilder(100000); - String line; try (BufferedReader in = new BufferedReader(new InputStreamReader( - zipped ? new GZIPInputStream(connection.getInputStream()) : connection.getInputStream(), "UTF-8"))) + zipped_ ? new GZIPInputStream(hr.body()) : hr.body(), "UTF-8"))) { - while ((line = in.readLine()) != null) - { - buffer.append(line); - buffer.append("\n"); - } + response = in.lines().collect(Collectors.joining("\n")); } - temp = buffer.toString(); - - // check for recoverable errors - - // rate limit (though might be a long one e.g. email) - if (temp.contains("error code=\"ratelimited\"")) + + // Check for rate limit (though might be a long one e.g. email) + if (response.contains("error code=\"ratelimited\"")) { + // the Retry-After header field is useless here + // see https://phabricator.wikimedia.org/T172293 log(Level.WARNING, caller, "Server-side throttle hit."); + Thread.sleep(10000); throw new HttpRetryException("Action throttled.", 503); } - // database lock - if (temp.contains("error code=\"readonly\"")) + // Check for database lock + if (response.contains("error code=\"readonly\"")) { log(Level.WARNING, caller, "Database locked!"); + Thread.sleep(10000); throw new HttpRetryException("Database locked!", 503); } - return temp; + + // No need to retry anymore, success or unrecoverable failure. + tries = 0; } catch (IOException ex) { + // Exception deliberately ignored until retries are depleted. if (tries == 0) throw ex; - try - { - Thread.sleep(10000); - } - catch (InterruptedException ignored) - { - } + } + catch (InterruptedException ignored) + { } } - while (temp.isEmpty()); - throw new AssertionError("Unreachable."); + while (tries != 0); + + // empty response from server + if (response.isEmpty()) + throw new UnknownError("Received empty response from server!"); + return response; } /** - * Performs a multi-part HTTP POST. - * @param url the url to post to - * @param params the POST parameters. Supported types: UTF-8 text, byte[]. - * Text and parameter names must NOT be URL encoded. - * @param caller the caller of this method - * @return the server response - * @throws IOException if a network error occurs - * @see #post(java.lang.String, java.lang.String, java.lang.String) - * @see Multipart/form-data - * @since 0.27 + * Converts HTTP POST parameters to Strings. See {@link #makeApiCall(Map, + * Map, String)} for the description. + * @param param the parameter to convert + * @return that parameter, as a String + * @throws UnsupportedOperationException if param is not a supported data type + * @since 0.35 */ - protected String multipartPost(String url, Map params, String caller) throws IOException + private String convertToString(Object param) { - String temp = ""; - int tries = maxtries; - do + // TODO: Replace with type switch in JDK 11/12 + if (param instanceof String) + return (String)param; + else if (param instanceof StringBuilder || param instanceof Number) + return param.toString(); + else if (param instanceof String[]) + return String.join("|", (String[])param); + else if (param instanceof OffsetDateTime) { - logurl(url, caller); - tries--; - try - { - URLConnection connection = makeConnection(url); - String boundary = "----------NEXT PART----------"; - connection.setRequestProperty("Content-Type", "multipart/form-data; boundary=" + boundary); - setCookies(connection); - connection.setDoOutput(true); - connection.setConnectTimeout(CONNECTION_CONNECT_TIMEOUT_MSEC); - connection.setReadTimeout(CONNECTION_READ_TIMEOUT_MSEC); - connection.connect(); - - // write stuff to a local buffer - boundary = "--" + boundary + "\r\n"; - ByteArrayOutputStream bout = new ByteArrayOutputStream(); - try (DataOutputStream out = new DataOutputStream(bout)) - { - out.writeBytes(boundary); - - // write params - for (Map.Entry entry : params.entrySet()) - { - String name = entry.getKey(); - Object value = entry.getValue(); - out.writeBytes("Content-Disposition: form-data; name=\"" + name + "\"\r\n"); - if (value instanceof String) - { - out.writeBytes("Content-Type: text/plain; charset=UTF-8\r\n\r\n"); - out.write(((String)value).getBytes("UTF-8")); - } - else if (value instanceof byte[]) - { - out.writeBytes("Content-Type: application/octet-stream\r\n\r\n"); - out.write((byte[])value); - } - else - throw new UnsupportedOperationException("Unrecognized data type"); - out.writeBytes("\r\n"); - out.writeBytes(boundary); - } - out.writeBytes("--\r\n"); - } - try (OutputStream uout = connection.getOutputStream()) - { - // write the buffer to the URLConnection - uout.write(bout.toByteArray()); - } - - // check lag and retry - if (checkLag(connection)) - return multipartPost(url, params, caller); - - // done, read the response - grabCookies(connection); - String line; - StringBuilder buffer = new StringBuilder(100000); - try (BufferedReader in = new BufferedReader(new InputStreamReader( - zipped ? new GZIPInputStream(connection.getInputStream()) : connection.getInputStream(), "UTF-8"))) - { - while ((line = in.readLine()) != null) - { - buffer.append(line); - buffer.append("\n"); - } - } - temp = buffer.toString(); - - // check for recoverable errors - - // rate limit (though might be a long one e.g. email) - if (temp.contains("error code=\"ratelimited\"")) - { - log(Level.WARNING, caller, "Server-side throttle hit."); - throw new HttpRetryException("Action throttled.", 503); - } - // database lock - if (temp.contains("error code=\"readonly\"")) - { - log(Level.WARNING, caller, "Database locked!"); - throw new HttpRetryException("Database locked!", 503); - } - return temp; - } - catch (IOException ex) - { - if (tries == 0) - throw ex; - try - { - Thread.sleep(10000); - } - catch (InterruptedException ignored) - { - } - } + OffsetDateTime date = (OffsetDateTime)param; + // https://www.mediawiki.org/wiki/Timestamp + // https://github.com/MER-C/wiki-java/issues/170 + return date.atZoneSameInstant(ZoneOffset.UTC) + .truncatedTo(ChronoUnit.MICROS) + .format(DateTimeFormatter.ISO_OFFSET_DATE_TIME); } - while (temp.isEmpty()); - throw new AssertionError("Unreachable."); + else if (param instanceof Collection) + { + Collection coll = (Collection)param; + return coll.stream() + .map(item -> convertToString(item)) + .collect(Collectors.joining("|")); + } + else + throw new UnsupportedOperationException("Unrecognized data type"); } - + /** - * Checks for database lag and sleeps if lag > maxlag. - * @param connection the URL connection used in the request + * Checks for database lag and sleeps if {@code lag < getMaxLag()}. + * @param response the HTTP response received * @return true if there was sufficient database lag. + * @throws InterruptedException if any wait was interrupted + * @see #getMaxLag() + * @see + * MediaWiki documentation * @since 0.32 */ - protected synchronized boolean checkLag(URLConnection connection) + protected synchronized boolean checkLag(HttpResponse response) throws InterruptedException { - int lag = connection.getHeaderFieldInt("X-Database-Lag", -5); + HttpHeaders hdrs = response.headers(); + long lag = hdrs.firstValueAsLong("X-Database-Lag").orElse(-5); // X-Database-Lag is the current lag rounded down to the nearest integer. // Thus, we need to retry in case of equality. if (lag >= maxlag) { - try - { - int time = connection.getHeaderFieldInt("Retry-After", 10); - logger.log(Level.WARNING, "Current database lag {0} s exceeds maxlag of {1} s, waiting {2} s.", new Object[] { lag, maxlag, time }); - Thread.sleep(time * 1000); - } - catch (InterruptedException ignored) - { - } + long time = hdrs.firstValueAsLong("Retry-After").orElse(10); + logger.log(Level.WARNING, "Current database lag {0} s exceeds maxlag of {1} s, waiting {2} s.", new Object[] { lag, maxlag, time }); + Thread.sleep(time * 1000L); return true; } return false; } /** - * Creates a new URL connection. Override to change SSL handling, use a - * proxy, etc. + * Sets the HTTPClient used by this instance. Use this to set a proxy, + * SSL parameters and the connection timeout. + * @param builder a HTTP request builder + * @since 0.37 + */ + public void setHttpClient(HttpClient.Builder builder) + { + client = builder.cookieHandler(cookies).build(); + } + + /** + * Creates a new HTTP request. Override to change request properties. * @param url a URL string - * @return a connection to that URL + * @return a HTTP request builder for that URL * @throws IOException if a network error occurs * @since 0.31 */ - protected URLConnection makeConnection(String url) throws IOException + protected HttpRequest.Builder makeConnection(String url) throws IOException { - return new URL(url).openConnection(); + return HttpRequest.newBuilder(URI.create(url)) + .timeout(Duration.ofMillis(read_timeout_msec)) + .header("User-Agent", useragent) + .header("Accept-encoding", "gzip"); } + /** + * Checks for errors from standard read/write requests and throws the + * appropriate unchecked exception. + * + * @param response the response from the server to analyze + * @param errors additional errors to check for where throwing an unchecked + * exception is the desired behavior (function is of MediaWiki error + * message) + * @param warnings additional errors to check for where throwing an exception + * is not required (function is of MediaWiki error message) + * @throws RuntimeException or subclasses depending on the type of errors + * checked for + * @return whether the action was successful + * @since 0.37 + */ + protected boolean detectUncheckedErrors(String response, Map> errors, + Map> warnings) + { + if (response.contains("> uncheckederrors, + Map> info) throws IOException, LoginException { // perform various status checks every 100 or so edits if (statuscounter > statusinterval) { // purge user rights in case of desysop or loss of other priviliges - user.getUserInfo(); + user = getUsers(List.of(user.getUsername())).get(0); if ((assertion & ASSERT_SYSOP) == ASSERT_SYSOP && !user.isA("sysop")) // assert user.isA("sysop") : "Sysop privileges missing or revoked, or session expired"; throw new AssertionError("Sysop privileges missing or revoked, or session expired"); @@ -7358,35 +8408,36 @@ protected void checkErrorsAndUpdateStatus(String line, String caller) throws IOE else statuscounter++; - // successful - if (line.contains("result=\"Success\"")) - return; - // empty response from server - if (line.isEmpty()) - throw new UnknownError("Received empty response from server!"); - // assertions - if ((assertion & ASSERT_BOT) == ASSERT_BOT && line.contains("error code=\"assertbotfailed\"")) - // assert !line.contains("error code=\"assertbotfailed\"") : "Bot privileges missing or revoked, or session expired."; - throw new AssertionError("Bot privileges missing or revoked, or session expired."); - if ((assertion & ASSERT_USER) == ASSERT_USER && line.contains("error code=\"assertuserfailed\"")) - // assert !line.contains("error code=\"assertuserfailed\"") : "Session expired."; - throw new AssertionError("Session expired."); - // protected page errors - if (line.matches("(protectednamespace|customcssjsprotected|cascadeprotected|protectedpage|protectedtitle)")) - throw new CredentialNotFoundException("Page is protected."); - if (line.contains("error code=\"permissiondenied\"")) - throw new CredentialNotFoundException("Permission denied."); // session expired or stupidity - // blocked! (note here the \" in blocked is deliberately missing for emailUser() - if (line.contains("error code=\"blocked") || line.contains("error code=\"autoblocked\"")) - { - log(Level.SEVERE, caller, "Cannot " + caller + " - user is blocked!."); - throw new AccountLockedException("Current user is blocked!"); - } - // unknown error - if (line.contains("error code=\"unknownerror\"")) - throw new UnknownError("Unknown MediaWiki API error, response was " + line); - // generic (automatic retry) - throw new IOException("MediaWiki error, response was " + line); + if (!line.contains(" constructRevisionString(long[] ids) { // sort and remove duplicates per https://mediawiki.org/wiki/API String[] sortedids = Arrays.stream(ids) .distinct() + .filter(id -> id >= 0) .sorted() .mapToObj(String::valueOf) .toArray(String[]::new); - + StringBuilder buffer = new StringBuilder(); - ArrayList chunks = new ArrayList<>(); + List chunks = new ArrayList<>(); for (int i = 0; i < sortedids.length; i++) { buffer.append(sortedids[i]); - if (i == ids.length - 1 || i == slowmax - 1) + if (i == ids.length - 1 || (i % slowmax) == slowmax - 1) { chunks.add(buffer.toString()); buffer.setLength(0); } else - buffer.append("%7C"); + buffer.append('|'); } - return chunks.toArray(new String[chunks.size()]); + return chunks; } /** * Cuts up a list of titles into batches for prop=X&titles=Y type queries. - * @param lengthBaseUrl the length of the url when no titles are present * @param titles a list of titles. - * @param limit whether to apply the maximum URL size * @return the titles ready for insertion into a URL - * @throws IOException if a network error occurs * @since 0.29 */ - protected String[] constructTitleString(int lengthBaseUrl, String[] titles, boolean limit) throws IOException + protected List constructTitleString(List titles) { - // sort and remove duplicates per https://mediawiki.org/wiki/API - Set set = new TreeSet<>(); - for (String title : titles) - set.add(encode(title, true)); - String[] titlesEnc = set.toArray(new String[set.size()]); - + // sort and remove duplicates + List titles_unique = titles.stream().sorted().distinct().collect(Collectors.toList()); + // actually construct the string - String titleStringToken = encode("|", false); ArrayList ret = new ArrayList<>(); - StringBuilder buffer = new StringBuilder(); - buffer.append(titlesEnc[0]); - int num = 1; - for (int i = num; i < titlesEnc.length; i++) + for (int i = 0; i < titles_unique.size() / slowmax + 1; i++) { - if (num < slowmax && - buffer.length() + titleStringToken.length() + titlesEnc[i].length() < URL_LENGTH_LIMIT - lengthBaseUrl) - { - buffer.append(titleStringToken); - } - else - { - ret.add(buffer.toString()); - buffer.setLength(0); - num = 0; - } - buffer.append(titlesEnc[i]); - ++num; + ret.add(String.join("|", + titles_unique.subList(i * slowmax, Math.min(titles_unique.size(), (i + 1) * slowmax)))); } - ret.add(buffer.toString()); - return ret.toArray(new String[ret.size()]); - } - - /** - * UTF-8 encode the String with URLEncoder after optional normalization; - * Usually, normalize should be set to true when a title or name String is - * passed in as an argument of a method. - * - * @param text the text to encode - * @param normalize if the text should be normalized first - * @return the encoded text - * @throws IOException if a network error occurs during initialization of the namespaces - */ - private String encode(String text, boolean normalize) throws IOException - { - final String encoding = "UTF-8"; - if (normalize) - text = normalize(text); - return URLEncoder.encode(text, encoding); + return ret; } /** * Convenience method for normalizing MediaWiki titles. (Converts all * underscores to spaces, localizes namespace names, fixes case of first - * char and does some other unicode fixes). + * char and does some other unicode fixes). Beware that this will not + * produce the same results as server-side normalization in a few corner + * cases, most notably: HTML entities and gender distinction in the user + * namespace prefix. * @param s the string to normalize * @return the normalized string * @throws IllegalArgumentException if the title is invalid - * @throws IOException if a network error occurs + * @throws UncheckedIOException if the namespace cache has not been + * populated, and a network error occurs when populating it * @since 0.27 */ - public String normalize(String s) throws IOException + public String normalize(String s) { + // remove section names + if (s.contains("#")) + s = s.substring(0, s.indexOf("#")); // remove leading colon if (s.startsWith(":")) s = s.substring(1); + s = s.replace('_', ' ').trim(); if (s.isEmpty()) - return s; + throw new IllegalArgumentException("Empty or whitespace only title."); int ns = namespace(s); // localize namespace names if (ns != MAIN_NAMESPACE) { - int colon = s.indexOf(":"); + int colon = s.indexOf(':'); s = namespaceIdentifier(ns) + s.substring(colon); } char[] temp = s.toCharArray(); @@ -7583,7 +8600,7 @@ public String normalize(String s) throws IOException { switch (temp[i]) { - // illegal characters + // illegal characters case '{': case '}': case '<': @@ -7592,13 +8609,10 @@ public String normalize(String s) throws IOException case ']': case '|': throw new IllegalArgumentException(s + " is an illegal title"); - case '_': - temp[i] = ' '; - break; } } // https://mediawiki.org/wiki/Unicode_normalization_considerations - String temp2 = new String(temp).trim().replaceAll("\\s+", " "); + String temp2 = new String(temp).replaceAll("\\s+", " "); return Normalizer.normalize(temp2, Normalizer.Form.NFC); } @@ -7607,7 +8621,7 @@ public String normalize(String s) throws IOException * and other write actions. * @since 0.30 */ - private synchronized void throttle() + protected synchronized void throttle() { try { @@ -7627,7 +8641,7 @@ private synchronized void throttle() * Checks whether the currently logged on user has sufficient rights to * edit/move a protected page. * - * @param pageinfo the output from {@link #getPageInfo(java.lang.String)} + * @param pageinfo the output from {@link #getPageInfo} for an article * @param action what we are doing * @return whether the user can perform the specified action * @throws IOException if a network error occurs @@ -7644,57 +8658,11 @@ protected boolean checkRights(Map pageinfo, String action) throw if (level.equals(FULL_PROTECTION)) return user.isAllowedTo("editprotected"); } - if ((Boolean)protectionstate.get("cascade") == Boolean.TRUE) // can be null + if (Boolean.TRUE.equals(protectionstate.get("cascade"))) return user.isAllowedTo("editprotected"); return true; } - // cookie methods - - /** - * Sets cookies to an unconnected URLConnection and enables gzip - * compression of returned text. - * @param u an unconnected URLConnection - */ - protected void setCookies(URLConnection u) - { - StringBuilder cookie = new StringBuilder(100); - for (Map.Entry entry : cookies.entrySet()) - { - cookie.append(entry.getKey()); - cookie.append("="); - cookie.append(entry.getValue()); - cookie.append("; "); - } - u.setRequestProperty("Cookie", cookie.toString()); - - // enable gzip compression - if (zipped) - u.setRequestProperty("Accept-encoding", "gzip"); - u.setRequestProperty("User-Agent", useragent); - } - - /** - * Grabs cookies from the URL connection provided. - * @param u an unconnected URLConnection - */ - private void grabCookies(URLConnection u) - { - String headerName; - for (int i = 1; (headerName = u.getHeaderFieldKey(i)) != null; i++) - if (headerName.equalsIgnoreCase("Set-Cookie")) - { - String cookie = u.getHeaderField(i); - cookie = cookie.substring(0, cookie.indexOf(';')); - String name = cookie.substring(0, cookie.indexOf('=')); - String value = cookie.substring(cookie.indexOf('=') + 1, cookie.length()); - // these cookies were pruned, but are still sent for some reason? - // TODO: when these cookies are no longer sent, remove this test - if (!value.equals("deleted")) - cookies.put(name, value); - } - } - // logging methods /** @@ -7719,30 +8687,4 @@ protected void logurl(String url, String method) { logger.logp(Level.INFO, "Wiki", method, "Fetching URL {0}", url); } - - // serialization - - /** - * Writes this wiki to a file. - * @param out an ObjectOutputStream to write to - * @throws IOException if there are local IO problems - * @since 0.10 - */ - private void writeObject(ObjectOutputStream out) throws IOException - { - out.defaultWriteObject(); - } - - /** - * Reads a copy of a wiki from a file. - * @param in an ObjectInputStream to read from - * @throws IOException if there are local IO problems - * @throws ClassNotFoundException if we can't recognize the input - * @since 0.10 - */ - private void readObject(ObjectInputStream in) throws IOException, ClassNotFoundException - { - in.defaultReadObject(); - statuscounter = statusinterval; // force a status check on next edit - } } diff --git a/src/pattypan/Main.java b/src/pattypan/Main.java index f44ce93..4724a19 100644 --- a/src/pattypan/Main.java +++ b/src/pattypan/Main.java @@ -84,7 +84,7 @@ public static void main(String[] args) { new String[]{os, Settings.VERSION} ); - Session.WIKI = new Wiki(wiki, scriptPath, protocol); + Session.WIKI = Wiki.newSession(wiki, scriptPath, protocol); launch(args); } } diff --git a/src/pattypan/Session.java b/src/pattypan/Session.java index 6ae27a4..073d65e 100644 --- a/src/pattypan/Session.java +++ b/src/pattypan/Session.java @@ -51,7 +51,7 @@ public final class Session { public static String WIKICODE = ""; public static ArrayList VARIABLES = new ArrayList<>(Arrays.asList("path", "name")); - public static Wiki WIKI = new Wiki("commons.wikimedia.org"); + public static Wiki WIKI = Wiki.newSession("commons.wikimedia.org"); public static ArrayList FILES_TO_UPLOAD = new ArrayList<>(); static { diff --git a/src/pattypan/panes/UploadPane.java b/src/pattypan/panes/UploadPane.java index 4397a64..48bad19 100644 --- a/src/pattypan/panes/UploadPane.java +++ b/src/pattypan/panes/UploadPane.java @@ -27,6 +27,7 @@ import java.net.UnknownHostException; import java.util.ArrayList; import java.util.Map; +import java.util.List; import java.util.logging.Level; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -224,7 +225,7 @@ private String getMediaWikiError(Exception error) { */ private boolean isFileNameTaken(String name) { try { - Map map = Session.WIKI.getPageInfo("File:" + name); + Map map = Session.WIKI.getPageInfo(List.of("File:" + name)).get(0); return (boolean) map.get("exists"); } catch (UnknownHostException ex) { Session.LOGGER.log(Level.WARNING, @@ -240,29 +241,6 @@ private boolean isFileNameTaken(String name) { return false; } } - - private String isFileUploaded(UploadElement ue) { - String checksum = ue.getFileChecksum(); - try { - Wiki.LogEntry[] entries = Session.WIKI.getChecksumDuplicates(checksum); - if(entries.length > 0) { - return entries[0].getTarget(); - } - return null; - } catch (UnknownHostException ex) { - Session.LOGGER.log(Level.WARNING, - "Error occurred during file SHA1 check: {0}", - new String[]{"no internet connection"} - ); - return null; - } catch (IOException ex) { - Session.LOGGER.log(Level.WARNING, - "Error occurred during file SHA1 check: {0}", - new String[]{ex.getLocalizedMessage()} - ); - return null; - } - } private void uploadFiles(ArrayList fileList) { Task task = new Task() { @@ -290,17 +268,6 @@ protected Object call() { skipped++; continue; } - - String duplicate = isFileUploaded(ue); - if (duplicate != null) { - updateMessage(String.format( - "FILE_DUPLICATE | %s | %s | %s", - current + 1, name, duplicate - )); - Thread.sleep(500); - skipped++; - continue; - } if (ue.getData("path").startsWith("https://") || ue.getData("path").startsWith("http://")) { Session.WIKI.upload(ue.getUrl(), name, ue.getWikicode(), summary); From bc2887efd090be75d9e650c23022adaa82723aee Mon Sep 17 00:00:00 2001 From: Abbe98 Date: Fri, 15 Oct 2021 16:20:52 +0200 Subject: [PATCH 06/18] simplify build system --- .gitignore | 2 + README.md | 10 +- build.xml | 101 +- manifest.mf | 3 - nbproject/build-impl.xml | 1404 ------ nbproject/configs/Run_as_WebStart.properties | 2 - nbproject/configs/Run_in_Browser.properties | 2 - nbproject/genfiles.properties | 8 - nbproject/jfx-impl.xml | 4049 ------------------ nbproject/project.properties | 136 - nbproject/project.xml | 23 - 11 files changed, 33 insertions(+), 5707 deletions(-) delete mode 100644 manifest.mf delete mode 100644 nbproject/build-impl.xml delete mode 100644 nbproject/configs/Run_as_WebStart.properties delete mode 100644 nbproject/configs/Run_in_Browser.properties delete mode 100644 nbproject/genfiles.properties delete mode 100644 nbproject/jfx-impl.xml delete mode 100644 nbproject/project.properties delete mode 100644 nbproject/project.xml diff --git a/.gitignore b/.gitignore index 513ccb1..0fe5d92 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ store dist nbproject\private /nbproject/private/ +bin +pattypan.jar diff --git a/README.md b/README.md index 7fcec75..186cfbd 100644 --- a/README.md +++ b/README.md @@ -9,18 +9,12 @@ __[:arrow_down: Download](https://github.com/yarl/pattypan/releases)__ ---- ### Build and run -Program is being written using [NetBeans IDE](https://netbeans.org/) and [Apache Ant](https://ant.apache.org/) is used for building. In order to download and build source code, do following: +[Apache Ant](https://ant.apache.org/) is used for building Pattypan. In order to download and build source code, do the following: ``` git clone https://github.com/yarl/pattypan.git cd pattypan -ant package-for-store -``` -You will find compiled `.jar` file in `store` directory. - -``` -cd store -java -jar pattypan.jar +ant compile && ant jar ``` You can also set test server or any other server: diff --git a/build.xml b/build.xml index 14e0102..ae3c141 100644 --- a/build.xml +++ b/build.xml @@ -1,79 +1,36 @@ - - Builds, tests, and runs the project pattypan. - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - + + diff --git a/manifest.mf b/manifest.mf deleted file mode 100644 index 328e8e5..0000000 --- a/manifest.mf +++ /dev/null @@ -1,3 +0,0 @@ -Manifest-Version: 1.0 -X-COMMENT: Main-Class will be added automatically by build - diff --git a/nbproject/build-impl.xml b/nbproject/build-impl.xml deleted file mode 100644 index 5dc16bb..0000000 --- a/nbproject/build-impl.xml +++ /dev/null @@ -1,1404 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Must set src.dir - Must set build.dir - Must set dist.dir - Must set build.classes.dir - Must set dist.javadoc.dir - Must set build.test.classes.dir - Must set build.test.results.dir - Must set build.classes.excludes - Must set dist.jar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Must set javac.includes - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - No tests executed. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Must set JVM to use for profiling in profiler.info.jvm - Must set profiler agent JVM arguments in profiler.info.jvmargs.agent - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Must select some files in the IDE or set javac.includes - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - To run this application from the command line without Ant, try: - - java -jar "${dist.jar.resolved}" - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Must select one file in the IDE or set run.class - - - - Must select one file in the IDE or set run.class - - - - - - - - - - - - - - - - - - - - - - - Must select one file in the IDE or set debug.class - - - - - Must select one file in the IDE or set debug.class - - - - - Must set fix.includes - - - - - - - - - - This target only works when run from inside the NetBeans IDE. - - - - - - - - - Must select one file in the IDE or set profile.class - This target only works when run from inside the NetBeans IDE. - - - - - - - - - This target only works when run from inside the NetBeans IDE. - - - - - - - - - - - - - This target only works when run from inside the NetBeans IDE. - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Must select one file in the IDE or set run.class - - - - - - Must select some files in the IDE or set test.includes - - - - - Must select one file in the IDE or set run.class - - - - - Must select one file in the IDE or set applet.url - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Must select some files in the IDE or set javac.includes - - - - - - - - - - - - - - - - - - Some tests failed; see details above. - - - - - - - - - Must select some files in the IDE or set test.includes - - - - Some tests failed; see details above. - - - - Must select some files in the IDE or set test.class - Must select some method in the IDE or set test.method - - - - Some tests failed; see details above. - - - - - Must select one file in the IDE or set test.class - - - - Must select one file in the IDE or set test.class - Must select some method in the IDE or set test.method - - - - - - - - - - - - - - Must select one file in the IDE or set applet.url - - - - - - - - - Must select one file in the IDE or set applet.url - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/nbproject/configs/Run_as_WebStart.properties b/nbproject/configs/Run_as_WebStart.properties deleted file mode 100644 index 670fff0..0000000 --- a/nbproject/configs/Run_as_WebStart.properties +++ /dev/null @@ -1,2 +0,0 @@ -# Do not modify this property in this configuration. It can be re-generated. -$label=Run as WebStart diff --git a/nbproject/configs/Run_in_Browser.properties b/nbproject/configs/Run_in_Browser.properties deleted file mode 100644 index f2a5a65..0000000 --- a/nbproject/configs/Run_in_Browser.properties +++ /dev/null @@ -1,2 +0,0 @@ -# Do not modify this property in this configuration. It can be re-generated. -$label=Run in Browser diff --git a/nbproject/genfiles.properties b/nbproject/genfiles.properties deleted file mode 100644 index 0a722af..0000000 --- a/nbproject/genfiles.properties +++ /dev/null @@ -1,8 +0,0 @@ -build.xml.data.CRC32=a1c9267a -build.xml.script.CRC32=280de569 -build.xml.stylesheet.CRC32=8064a381@1.80.1.48 -# This file is used by a NetBeans-based IDE to track changes in generated files such as build-impl.xml. -# Do not edit this file. You may delete it but then the IDE will never regenerate such files for you. -nbproject/build-impl.xml.data.CRC32=38fa065d -nbproject/build-impl.xml.script.CRC32=d7c5f806 -nbproject/build-impl.xml.stylesheet.CRC32=830a3534@1.80.1.48 diff --git a/nbproject/jfx-impl.xml b/nbproject/jfx-impl.xml deleted file mode 100644 index 575cf06..0000000 --- a/nbproject/jfx-impl.xml +++ /dev/null @@ -1,4049 +0,0 @@ - - - - - JavaFX-specific Ant calls - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - ${cssfileslist} - - - - - - - - - - - - - - - - - - - - - - - - self.addMappedName( - (source.indexOf("jfxrt.jar") >= 0) || - (source.indexOf("deploy.jar") >= 0) || - (source.indexOf("javaws.jar") >= 0) || - (source.indexOf("plugin.jar") >= 0) - ? "" : source - ); - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/nbproject/project.properties b/nbproject/project.properties deleted file mode 100644 index 135e237..0000000 --- a/nbproject/project.properties +++ /dev/null @@ -1,136 +0,0 @@ -annotation.processing.enabled=true -annotation.processing.enabled.in.editor=false -annotation.processing.processors.list= -annotation.processing.run.all.processors=true -annotation.processing.source.output=${build.generated.sources.dir}/ap-source-output -application.title=pattypan -application.vendor=Pawel -build.classes.dir=${build.dir}/classes -build.classes.excludes=**/*.java,**/*.form -# This directory is removed when the project is cleaned: -build.dir=build -build.generated.dir=${build.dir}/generated -build.generated.sources.dir=${build.dir}/generated-sources -# Only compile against the classpath explicitly listed here: -build.sysclasspath=ignore -build.test.classes.dir=${build.dir}/test/classes -build.test.results.dir=${build.dir}/test/results -compile.on.save=true -compile.on.save.unsupported.javafx=true -# Uncomment to specify the preferred debugger connection transport: -#debug.transport=dt_socket -debug.classpath=\ - ${run.classpath} -debug.test.classpath=\ - ${run.test.classpath} -# This directory is removed when the project is cleaned: -dist.dir=dist -dist.jar=${dist.dir}/pattypan.jar -dist.javadoc.dir=${dist.dir}/javadoc -endorsed.classpath= -excludes= -file.reference.BrowserLauncher2-1_3.jar=lib/BrowserLauncher2-1_3.jar -file.reference.freemarker.jar=lib/freemarker.jar -file.reference.gson-2.8.5.jar=lib/gson-2.8.5.jar -file.reference.jxl.jar=lib/jxl.jar -file.reference.metadata-extractor-2.11.0.jar=lib/metadata-extractor-2.11.0.jar -file.reference.pattypan-src=src -file.reference.xmpcore-5.1.3.jar=lib/xmpcore-5.1.3.jar -includes=** -# Non-JavaFX jar file creation is deactivated in JavaFX 2.0+ projects -jar.archive.disabled=true -jar.compress=false -javac.classpath=\ - ${javafx.classpath.extension}:\ - ${file.reference.BrowserLauncher2-1_3.jar}:\ - ${file.reference.freemarker.jar}:\ - ${file.reference.gson-2.8.5.jar}:\ - ${file.reference.jxl.jar}:\ - ${file.reference.metadata-extractor-2.11.0.jar}:\ - ${file.reference.xmpcore-5.1.3.jar} -# Space-separated list of extra javac options -javac.compilerargs= -javac.deprecation=false -javac.external.vm=false -javac.processorpath=\ - ${javac.classpath} -javac.source=1.8 -javac.target=1.8 -javac.test.classpath=\ - ${javac.classpath}:\ - ${build.classes.dir} -javac.test.processorpath=\ - ${javac.test.classpath} -javadoc.additionalparam= -javadoc.author=false -javadoc.encoding=${source.encoding} -javadoc.noindex=false -javadoc.nonavbar=false -javadoc.notree=false -javadoc.private=false -javadoc.splitindex=true -javadoc.use=true -javadoc.version=false -javadoc.windowtitle= -javafx.application.implementation.version=1.0 -javafx.binarycss=false -javafx.classpath.extension=\ - ${java.home}/lib/javaws.jar:\ - ${java.home}/lib/deploy.jar:\ - ${java.home}/lib/plugin.jar -javafx.deploy.adddesktopshortcut=false -javafx.deploy.addstartmenushortcut=false -javafx.deploy.allowoffline=true -# If true, application update mode is set to 'background', if false, update mode is set to 'eager' -javafx.deploy.backgroundupdate=false -javafx.deploy.disable.proxy=false -javafx.deploy.embedJNLP=true -javafx.deploy.includeDT=true -javafx.deploy.installpermanently=false -javafx.deploy.permissionselevated=false -# Set true to prevent creation of temporary copy of deployment artifacts before each run (disables concurrent runs) -javafx.disable.concurrent.runs=false -# Set true to enable multiple concurrent runs of the same WebStart or Run-in-Browser project -javafx.enable.concurrent.external.runs=false -# This is a JavaFX project -javafx.enabled=true -javafx.fallback.class=com.javafx.main.NoJavaFXFallback -# Main class for JavaFX -javafx.main.class=pattypan.Main -javafx.preloader.class= -# This project does not use Preloader -javafx.preloader.enabled=false -javafx.preloader.jar.filename= -javafx.preloader.jar.path= -javafx.preloader.project.path= -javafx.preloader.type=none -# Set true for GlassFish only. Rebases manifest classpaths of JARs in lib dir. Not usable with signed JARs. -javafx.rebase.libs=false -javafx.run.height=600 -javafx.run.width=800 -javafx.signing.blob=false -javafx.signing.enabled=false -javafx.signing.type=notsigned -# Pre-JavaFX 2.0 WebStart is deactivated in JavaFX 2.0+ projects -jnlp.enabled=false -# Main class for Java launcher -main.class=com.javafx.main.Main -# For improved security specify narrower Codebase manifest attribute to prevent RIAs from being repurposed -manifest.custom.codebase=* -# Specify Permissions manifest attribute to override default (choices: sandbox, all-permissions) -manifest.custom.permissions= -manifest.file=manifest.mf -meta.inf.dir=${src.dir}/META-INF -mkdist.disabled=false -native.bundling.enabled=false -platform.active=default_platform -run.classpath=\ - ${dist.jar}:\ - ${javac.classpath}:\ - ${build.classes.dir} -run.jvmargs=-Duser.language=pl -Duser.country=PL -run.test.classpath=\ - ${javac.test.classpath}:\ - ${build.test.classes.dir} -source.encoding=UTF-8 -src.dir=${file.reference.pattypan-src} diff --git a/nbproject/project.xml b/nbproject/project.xml deleted file mode 100644 index f09ae0c..0000000 --- a/nbproject/project.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - org.netbeans.modules.java.j2seproject - - - - - - - - - - - - - pattypan - - - - - - - From c1f8d02f8da03aaf4f7a0f28b78d8f03594a2a69 Mon Sep 17 00:00:00 2001 From: Abbe98 Date: Fri, 29 Oct 2021 13:23:04 +0200 Subject: [PATCH 07/18] bump Wiki.java to resolve https://github.com/MER-C/wiki-java/issues/184 --- src/org/wikipedia/Wiki.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/org/wikipedia/Wiki.java b/src/org/wikipedia/Wiki.java index 14aa8d2..65fea8c 100644 --- a/src/org/wikipedia/Wiki.java +++ b/src/org/wikipedia/Wiki.java @@ -8128,13 +8128,11 @@ public String makeApiCall(Map getparams, Map pos // now we know how we're sending it, construct the post body if (multipart) { - byte[] nextpart = ("--" + boundary + "\r\n\"Content-Disposition: form-data; name=\\\"\"") - .getBytes(StandardCharsets.UTF_8); + String nextpart = "--" + boundary + "\r\nContent-Disposition: form-data; name=\""; for (Map.Entry entry : postparams.entrySet()) { - multipartPostBody.add(nextpart); Object value = entry.getValue(); - multipartPostBody.add((entry.getKey() + "\"\r\n").getBytes(StandardCharsets.UTF_8)); + multipartPostBody.add((nextpart + entry.getKey() + "\"").getBytes(StandardCharsets.UTF_8)); if (value instanceof String) multipartPostBody.add(("Content-Type: text/plain; charset=UTF-8\r\n\r\n" + (String)value + "\r\n") .getBytes(StandardCharsets.UTF_8)); From 956dd867145d2bb94b2798efb119504ba097a02b Mon Sep 17 00:00:00 2001 From: Abbe98 Date: Sun, 23 Jan 2022 13:41:23 +0100 Subject: [PATCH 08/18] bump wiki.java to resolve http upload error --- src/org/wikipedia/Wiki.java | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/src/org/wikipedia/Wiki.java b/src/org/wikipedia/Wiki.java index 65fea8c..6b864b8 100644 --- a/src/org/wikipedia/Wiki.java +++ b/src/org/wikipedia/Wiki.java @@ -1349,6 +1349,24 @@ public Map getSiteStatistics() throws IOException ret.put("jobs", Integer.parseInt(parseAttribute(text, "jobs", 0))); // job queue length return ret; } + + /** + * Require the given extension be installed on this wiki, or throw an + * UnsupportedOperationException if it isn't. + * @param extension the name of the extension to check + * @throws UnsupportedOperationException if that extension is not + * installed on this wiki + * @throws UncheckedIOException if the site info cache is not populated + * and a network error occurs when populating it + * @since 0.37 + */ + public void requiresExtension(String extension) + { + if (!installedExtensions().contains(extension)) + throw new UnsupportedOperationException("Extension \"" + extension + + "\" is not installed on " + getDomain() + ". " + + "Please check the extension name and [[Special:Version]]."); + } /** * Renders the specified wiki markup as HTML by passing it to the MediaWiki @@ -8110,7 +8128,7 @@ public String makeApiCall(Map getparams, Map pos boolean isPOST = (postparams != null && !postparams.isEmpty()); StringBuilder stringPostBody = new StringBuilder(); boolean multipart = false; - ArrayList multipartPostBody = new ArrayList<>(); + ByteArrayOutputStream multipartPostBody = new ByteArrayOutputStream(); String boundary = "----------NEXT PART----------"; if (isPOST) { @@ -8132,18 +8150,18 @@ public String makeApiCall(Map getparams, Map pos for (Map.Entry entry : postparams.entrySet()) { Object value = entry.getValue(); - multipartPostBody.add((nextpart + entry.getKey() + "\"").getBytes(StandardCharsets.UTF_8)); + multipartPostBody.write((nextpart + entry.getKey() + "\"").getBytes(StandardCharsets.UTF_8)); if (value instanceof String) - multipartPostBody.add(("Content-Type: text/plain; charset=UTF-8\r\n\r\n" + (String)value + "\r\n") + multipartPostBody.write(("Content-Type: text/plain; charset=UTF-8\r\n\r\n" + (String)value + "\r\n") .getBytes(StandardCharsets.UTF_8)); else if (value instanceof byte[]) { - multipartPostBody.add("Content-Type: application/octet-stream\r\n\r\n".getBytes(StandardCharsets.UTF_8)); - multipartPostBody.add((byte[])value); - multipartPostBody.add("\r\n".getBytes(StandardCharsets.UTF_8)); + multipartPostBody.write("Content-Type: application/octet-stream\r\n\r\n".getBytes(StandardCharsets.UTF_8)); + multipartPostBody.write((byte[])value); + multipartPostBody.write("\r\n".getBytes(StandardCharsets.UTF_8)); } } - multipartPostBody.add((boundary + "--\r\n").getBytes(StandardCharsets.UTF_8)); + multipartPostBody.write((boundary + "--\r\n").getBytes(StandardCharsets.UTF_8)); } else { @@ -8171,7 +8189,7 @@ else if (value instanceof byte[]) if (isPOST) { if (multipart) - connection = connection.POST(HttpRequest.BodyPublishers.ofByteArrays(multipartPostBody)) + connection = connection.POST(HttpRequest.BodyPublishers.ofByteArray(multipartPostBody.toByteArray())) .header("Content-Type", "multipart/form-data; boundary=" + boundary); else connection = connection.POST(HttpRequest.BodyPublishers.ofString(stringPostBody.toString())) From efa614135968e1b205a27be914067d3be9669a5e Mon Sep 17 00:00:00 2001 From: Abbe98 Date: Fri, 28 Jan 2022 22:08:29 +0100 Subject: [PATCH 09/18] `ant compile` task now takes the path to the OpenJFX SDK So that one no longer need to manually insert the OpenJFX jars. --- README.md | 4 ++-- build.xml | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 186cfbd..3034c28 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,12 @@ __[:arrow_down: Download](https://github.com/yarl/pattypan/releases)__ ---- ### Build and run -[Apache Ant](https://ant.apache.org/) is used for building Pattypan. In order to download and build source code, do the following: +[Apache Ant](https://ant.apache.org/) is used for building Pattypan. You need to have JDK 11 or later installed as well as an download of OpenJFX. In order to download and build source code, do the following: ``` git clone https://github.com/yarl/pattypan.git cd pattypan -ant compile && ant jar +ant compile -DopenjfxSDKPath="/path-to-openjfx-sdk/" && ant jar ``` You can also set test server or any other server: diff --git a/build.xml b/build.xml index ae3c141..8acb18b 100644 --- a/build.xml +++ b/build.xml @@ -8,9 +8,12 @@ + + + From 48ea9ea11d1607018b10d7c74d55961584efd65a Mon Sep 17 00:00:00 2001 From: Abbe98 Date: Sat, 29 Jan 2022 15:31:37 +0100 Subject: [PATCH 10/18] bump wiki.java to resolve issue with uploading stopping at random images --- src/org/wikipedia/Wiki.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/org/wikipedia/Wiki.java b/src/org/wikipedia/Wiki.java index 6b864b8..a41f241 100644 --- a/src/org/wikipedia/Wiki.java +++ b/src/org/wikipedia/Wiki.java @@ -8161,7 +8161,7 @@ else if (value instanceof byte[]) multipartPostBody.write("\r\n".getBytes(StandardCharsets.UTF_8)); } } - multipartPostBody.write((boundary + "--\r\n").getBytes(StandardCharsets.UTF_8)); + multipartPostBody.write((nextpart + "--\r\n").getBytes(StandardCharsets.UTF_8)); } else { From d110cf618d6515f25abebaa0c1d76b97dc57ec57 Mon Sep 17 00:00:00 2001 From: Sebastian Berlin Date: Tue, 1 Feb 2022 14:39:01 +0100 Subject: [PATCH 11/18] Package JavaFX Adds JavaFX components to the .jar. Linux support only. --- README.md | 29 +++----------------------- build.xml | 42 +++++++++++++++++++++++++++----------- src/pattypan/Launcher.java | 11 ++++++++++ 3 files changed, 44 insertions(+), 38 deletions(-) create mode 100644 src/pattypan/Launcher.java diff --git a/README.md b/README.md index 3034c28..f8c2f35 100644 --- a/README.md +++ b/README.md @@ -9,12 +9,13 @@ __[:arrow_down: Download](https://github.com/yarl/pattypan/releases)__ ---- ### Build and run -[Apache Ant](https://ant.apache.org/) is used for building Pattypan. You need to have JDK 11 or later installed as well as an download of OpenJFX. In order to download and build source code, do the following: +[Apache Ant](https://ant.apache.org/) is used for building Pattypan. You need to have JDK 11 or later installed as well as [a download of OpenJFX](https://gluonhq.com/products/javafx/). In order to download and build source code, do the following: ``` git clone https://github.com/yarl/pattypan.git cd pattypan -ant compile -DopenjfxSDKPath="/path-to-openjfx-sdk/" && ant jar +export PATH_TO_FX=path-to-javafx-sdk/lib +ant compile && ant jar ``` You can also set test server or any other server: @@ -27,30 +28,6 @@ java -jar pattypan.jar wiki="test2.wikipedia.org" protocol="https://" scriptPath Please note, that on test server file upload may be disabled for regular users. Admin account is suggested, you can request rights [here](https://test.wikipedia.org/wiki/Wikipedia:Requests/Permissions). If you have problems with program running, check [article on project wiki](https://github.com/yarl/pattypan/wiki/Run). -### Additional information for Ubuntu, Debian and Fedora based distributions -These linux distributions may require additional ```openjfx``` package. After installing Java on your system, download and install the ```openjfx``` package. You can install it by running: - -``` -sudo apt-get install openjfx -``` - -on your terminal. -You can also check this link: https://pkgs.org/download/openjfx for more information on ```openjfx``` package for these distributions. - -Starting with Ubuntu 18.10 (and around the same time in Debian Sid), openjfx version 8 is no more shipped with the distribution. Though it is possible to force the openjfx to stay in version 8, and run pattypan, this is pretty much a hack. If you still wish to proceed, you can gain access to the Ubuntu 18.04 repository by adding the following line to your `/etc/apt/sources.list` file: - -``` -deb-src http://fr.archive.ubuntu.com/ubuntu/ bionic universe multiverse -``` - -You can then run these commands to perfom the installation and pin these versions: -``` -apt purge openjfx -apt install openjfx=8u161-b12-1ubuntu2 libopenjfx-jni=8u161-b12-1ubuntu2 libopenjfx-java=8u161-b12-1ubuntu2 -apt-mark hold openjfx libopenjfx-jni libopenjfx-java -``` - - ### License Copyright (c) 2016 Paweł Marynowski. diff --git a/build.xml b/build.xml index 8acb18b..c5ac5b0 100644 --- a/build.xml +++ b/build.xml @@ -1,5 +1,7 @@ + + @@ -8,31 +10,47 @@ - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - diff --git a/src/pattypan/Launcher.java b/src/pattypan/Launcher.java new file mode 100644 index 0000000..1e583b2 --- /dev/null +++ b/src/pattypan/Launcher.java @@ -0,0 +1,11 @@ +package pattypan; + +import javafx.application.Platform; + +public class Launcher { + public static void main(String[] args) { + // Initialise JavaFX. + Platform.startup(() -> {}); + Main.main(args); + } +} From a364ae62397bee596ccec391da63314b6ac75b40 Mon Sep 17 00:00:00 2001 From: Abbe98 Date: Tue, 1 Feb 2022 17:47:28 +0100 Subject: [PATCH 12/18] always clean the bin directory before compiling Pattypan to make it easier to compile platform specific jars without needing to think about cacheed files. --- build.xml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/build.xml b/build.xml index c5ac5b0..df1056d 100644 --- a/build.xml +++ b/build.xml @@ -9,7 +9,11 @@ - + + + + + From e5a7a006415d1b8016d9dec931dfdf27f5615b3e Mon Sep 17 00:00:00 2001 From: Albin Larsson Date: Thu, 3 Feb 2022 18:25:18 +0100 Subject: [PATCH 13/18] Package for Windows (#3) * add dependent DLLs * Grabs OpenJFX ZIPs from the base directory. Added clean target. Default runs all targets. * update readme * update .gitignore for the new build process Co-authored-by: Sebastian Berlin --- .gitignore | 6 +-- README.md | 9 +++-- build.xml | 114 ++++++++++++++++++++++++++++++++++++++++------------- 3 files changed, 95 insertions(+), 34 deletions(-) diff --git a/.gitignore b/.gitignore index 0fe5d92..12373f3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,5 @@ build -store -dist -nbproject\private -/nbproject/private/ bin +tmp pattypan.jar +*.zip diff --git a/README.md b/README.md index f8c2f35..83cedef 100644 --- a/README.md +++ b/README.md @@ -9,15 +9,18 @@ __[:arrow_down: Download](https://github.com/yarl/pattypan/releases)__ ---- ### Build and run -[Apache Ant](https://ant.apache.org/) is used for building Pattypan. You need to have JDK 11 or later installed as well as [a download of OpenJFX](https://gluonhq.com/products/javafx/). In order to download and build source code, do the following: +[Apache Ant](https://ant.apache.org/) is used for building Pattypan. You need to have JDK 11 or later installed as well as [a download of OpenJFX](https://gluonhq.com/products/javafx/) for each platform you want to support. In order to download and build source code, do the following: ``` git clone https://github.com/yarl/pattypan.git cd pattypan -export PATH_TO_FX=path-to-javafx-sdk/lib -ant compile && ant jar +ant ``` +This will run the default `build` target. It assumes that the current directory contains the OpenJFX SDK ZIP(s) and will unpack the required files to the correct locations. The resulting JAR will support Linux, Windows or both. The ZIPs present dictates what platforms will be supported. Note that the ZIPs should have their default name to be included. + +A temporary directory will be used during the build process and removed afterwards. It's default path is *tmp/* and can be set using `ant -Dtmp=...` + You can also set test server or any other server: ``` diff --git a/build.xml b/build.xml index df1056d..60acded 100644 --- a/build.xml +++ b/build.xml @@ -1,7 +1,7 @@ - + + - - + @@ -9,15 +9,90 @@ - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - + - + @@ -30,26 +105,6 @@ - - - - - - - - - - - - - - - - - - - - @@ -58,4 +113,9 @@ + + + + + From bfa752ac0b7fe9ad6edb4d0f2229f4eb4447d234 Mon Sep 17 00:00:00 2001 From: Abbe98 Date: Fri, 4 Feb 2022 10:42:42 +0100 Subject: [PATCH 14/18] Ensure text is rendered as UTF-8 Java <= 8 used ISO-8859-1. --- src/pattypan/Util.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/src/pattypan/Util.java b/src/pattypan/Util.java index 84b9186..2a9ddb5 100644 --- a/src/pattypan/Util.java +++ b/src/pattypan/Util.java @@ -32,7 +32,6 @@ import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; import java.net.URI; import java.net.URISyntaxException; import java.net.URL; @@ -63,12 +62,9 @@ private Util() {} public static String text(String key) { try { - String val = bundle.getString(key); - return new String(val.getBytes("ISO-8859-1"), "UTF-8"); + return bundle.getString(key); } catch (final MissingResourceException ex) { return ""; - } catch (UnsupportedEncodingException ex) { - return ""; } } From 5c658b9568fbb09e861cdaafe78b18f40ce9a913 Mon Sep 17 00:00:00 2001 From: Albin Larsson Date: Fri, 4 Feb 2022 12:05:12 +0100 Subject: [PATCH 15/18] Package for MacOS (#4) * package for MacOS --- build.xml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) diff --git a/build.xml b/build.xml index 60acded..6ca20df 100644 --- a/build.xml +++ b/build.xml @@ -86,6 +86,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + From 4b90414215a0286ce02572a85f529eea382b97cf Mon Sep 17 00:00:00 2001 From: sebastian-berlin-wmse <36327337+sebastian-berlin-wmse@users.noreply.github.com> Date: Fri, 4 Feb 2022 12:06:05 +0100 Subject: [PATCH 16/18] Remove unused library from JAR (#5) --- build.xml | 2 -- 1 file changed, 2 deletions(-) diff --git a/build.xml b/build.xml index 6ca20df..904adce 100644 --- a/build.xml +++ b/build.xml @@ -42,7 +42,6 @@ - @@ -78,7 +77,6 @@ - From 2d2f15d17536ae4ae8305764c74699fd85c9651d Mon Sep 17 00:00:00 2001 From: Sebastian Berlin Date: Fri, 4 Feb 2022 12:48:47 +0100 Subject: [PATCH 17/18] Fix freezing when clicking log link on Linux Solution found at https://stackoverflow.com/a/65863422. --- src/pattypan/Util.java | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pattypan/Util.java b/src/pattypan/Util.java index 84b9186..dbbc641 100644 --- a/src/pattypan/Util.java +++ b/src/pattypan/Util.java @@ -27,6 +27,7 @@ import edu.stanford.ejalbert.exception.BrowserLaunchingInitializingException; import edu.stanford.ejalbert.exception.UnsupportedOperatingSystemException; import java.awt.Desktop; +import java.awt.EventQueue; import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; @@ -103,11 +104,13 @@ public static void openUrl(String url) { } public static void openDirectory(Path path) { - try { - Desktop.getDesktop().open(new File(path.toString())); - } catch (IllegalArgumentException | IOException ex) { - Session.LOGGER.log(Level.WARNING, null, ex); - } + EventQueue.invokeLater(() -> { + try { + Desktop.getDesktop().open(new File(path.toString())); + } catch (IllegalArgumentException | IOException ex) { + Session.LOGGER.log(Level.WARNING, null, ex); + } + }); } /* row and column utils */ From ac58d5b630c744301f23ab03d1a673fd1a5afa16 Mon Sep 17 00:00:00 2001 From: Abbe98 Date: Mon, 7 Feb 2022 15:56:12 +0100 Subject: [PATCH 18/18] bump version --- src/pattypan/Settings.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/pattypan/Settings.java b/src/pattypan/Settings.java index 10882c2..78a3713 100644 --- a/src/pattypan/Settings.java +++ b/src/pattypan/Settings.java @@ -42,7 +42,7 @@ public final class Settings { private Settings() {}; public static final String NAME = "pattypan"; - public static final String VERSION = "21.10"; + public static final String VERSION = "22.02"; public static final String USERAGENT = NAME + "/" + VERSION + " (https://github.com/yarl/pattypan)"; public static final Map SETTINGS = new HashMap<>();