Merge pull request #99 from yarl/dev

version 19.05

nbproject/project.properties

@@ -31,11 +31,11 @@ endorsed.classpath=
 excludes=
 file.reference.BrowserLauncher2-1_3.jar=lib/BrowserLauncher2-1_3.jar
 file.reference.freemarker.jar=lib/freemarker.jar
-file.reference.gson-2.6.2.jar=lib/gson-2.6.2.jar
+file.reference.gson-2.8.5.jar=lib/gson-2.8.5.jar
 file.reference.jxl.jar=lib/jxl.jar
-file.reference.metadata-extractor-2.9.1.jar=lib/metadata-extractor-2.9.1.jar
+file.reference.metadata-extractor-2.11.0.jar=lib/metadata-extractor-2.11.0.jar
 file.reference.pattypan-src=src
-file.reference.xmpcore-5.1.2.jar=lib/xmpcore-5.1.2.jar
+file.reference.xmpcore-5.1.3.jar=lib/xmpcore-5.1.3.jar
 includes=**
 # Non-JavaFX jar file creation is deactivated in JavaFX 2.0+ projects
 jar.archive.disabled=true
@@ -44,10 +44,10 @@ javac.classpath=\
     ${javafx.classpath.extension}:\
     ${file.reference.BrowserLauncher2-1_3.jar}:\
     ${file.reference.freemarker.jar}:\
-    ${file.reference.gson-2.6.2.jar}:\
+    ${file.reference.gson-2.8.5.jar}:\
     ${file.reference.jxl.jar}:\
-    ${file.reference.metadata-extractor-2.9.1.jar}:\
-    ${file.reference.xmpcore-5.1.2.jar}
+    ${file.reference.metadata-extractor-2.11.0.jar}:\
+    ${file.reference.xmpcore-5.1.3.jar}
 # Space-separated list of extra javac options
 javac.compilerargs=
 javac.deprecation=false

src/org/wikipedia/Wiki.java

@@ -3678,7 +3678,7 @@ public LogEntry[] getUploads(User user) throws IOException
      *  @return a list of all live images the user has uploaded
      *  @throws IOException if a network error occurs
      */
-    public LogEntry[] getUploads(User user, OffsetDateTime start, OffsetDateTime end) throws IOException
+        public LogEntry[] getUploads(User user, OffsetDateTime start, OffsetDateTime end) throws IOException
     {
         StringBuilder url = new StringBuilder(query);
         url.append("list=allimages&aisort=timestamp&aiprop=timestamp%7Ccomment&aiuser=");
@@ -3713,6 +3713,30 @@ public LogEntry[] getUploads(User user, OffsetDateTime start, OffsetDateTime end
         log(Level.INFO, "getUploads", "Successfully retrieved uploads of " + user.getUsername() + " (" + size + " uploads)");
         return uploads.toArray(new LogEntry[size]);
     }
+
+    public LogEntry[] getChecksumDuplicates(String checksum) throws IOException
+    {
+        StringBuilder url = new StringBuilder(query);
+        url.append("list=allimages&aisha1=");
+        url.append(encode(checksum, false));
+
+        List<LogEntry> uploads = queryAPIResult("ai", url, "getChecksumDuplicates", (line, results) ->
+        {
+            for (int i = line.indexOf("<img "); i > 0; i = line.indexOf("<img ", ++i))
+            {
+                int b = line.indexOf("/>", i);
+                LogEntry le = parseLogEntry(line.substring(i, b));
+                le.type = UPLOAD_LOG;
+                le.action = "upload"; // unless it's an overwrite?
+                le.user = user;
+                results.add(le);
+            }
+        });
+
+        int size = uploads.size();
+        log(Level.INFO, "getChecksumDuplicates", "Successfully retrieved files based on checksum " + checksum);
+        return uploads.toArray(new LogEntry[size]);
+    }
 
     /**
      *  Uploads an image. Equivalent to [[Special:Upload]]. Supported

src/pattypan/Settings.java

@@ -148,6 +148,41 @@ public final class Settings {
                     + "</#if>"
             )
     );
+    TEMPLATES.put(
+            "Art Photo",
+            new Template("Art Photo",
+                    new TemplateField[]{
+                      new TemplateField("wikidata", "Wikidata"),
+                      new TemplateField("artwork_license", "Artwork license"),
+                      new TemplateField("photo_description", "Photo description"),
+                      new TemplateField("photo_date", "Photo date"),
+                      new TemplateField("photographer", "Photographer"),
+                      new TemplateField("source", "Source"),
+                      new TemplateField("photo_license", "Photo license"),
+                      new TemplateField("other_versions", "Other versions"),
+                      new TemplateField("partnership", "Partnership")
+                    }, "=={{int:filedesc}}==\n"
+                    + "{{Art Photo\n"
+                    + " |wikidata = ${wikidata}\n"
+                    + " |artwork license = ${artwork_license}\n"
+                    + " |photo description = ${photo_description}\n"
+                    + " |photo date = ${photo_date}\n"
+                    + " |photographer = ${photographer}\n"
+                    + " |source = ${source}\n"
+                    + " |photo license = ${photo_license}\n"
+                    + " |other_versions = ${other_versions}\n"
+                    + "}}\n\n"
+                    + "=={{int:license-header}}==\n"
+                    + "${artwork_license}${photo_license}${partnership}"
+                    + "\n\n"
+                    + "<#if categories ? has_content>\n"
+                    + "<#list categories ? split(\";\") as category>\n"
+                    + "[[Category:${category?trim}]]\n"
+                    + "</#list>\n"
+                    + "<#else>{{subst:unc}}\n"
+                    + "</#if>"
+            )
+    );
     TEMPLATES.put(
             "Book",
             new Template("Book",

src/pattypan/UploadElement.java

@@ -24,10 +24,14 @@
 package pattypan;
 
 import java.io.File;
+import java.io.IOException;
 import java.net.URL;
 import java.net.MalformedURLException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
 import java.util.Map;
 import java.util.logging.Level;
+import java.util.logging.Logger;
 
 public final class UploadElement {
 
@@ -58,6 +62,18 @@ public Map<String, String> getData() {
   public File getFile() {
     return new File(getData("path"));
   }
+
+  public String getFileChecksum() {
+    File file = this.getFile();
+    try {
+      MessageDigest shaDigest = MessageDigest.getInstance("SHA-1");
+      String shaChecksum = Util.getFileChecksum(shaDigest, file);
+      return shaChecksum;
+    } catch (NoSuchAlgorithmException | IOException e) {
+      Session.LOGGER.log(Level.SEVERE, null, e);
+      return null;
+    }
+  }
 
   public URL getUrl() {
     try {

src/pattypan/Util.java

@@ -29,14 +29,17 @@
 import java.awt.Desktop;
 import java.io.BufferedReader;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.io.UnsupportedEncodingException;
 import java.net.URI;
 import java.net.URISyntaxException;
 import java.net.URL;
 import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.security.MessageDigest;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.HashMap;
@@ -60,9 +63,12 @@ private Util() {}
 
   public static String text(String key) {
     try {
-      return bundle.getString(key);
+      String val = bundle.getString(key); 
+      return new String(val.getBytes("ISO-8859-1"), "UTF-8");
     } catch (final MissingResourceException ex) {
       return "";
+    } catch (UnsupportedEncodingException ex) {
+      return "";
     }
   }
 
@@ -130,9 +136,9 @@ public static ColumnConstraints newColumn(int value, String unit, HPos position)
   }
 
   /* file utils */
-  private final static ArrayList<String> allowedExtentionImage = new ArrayList<>(
+  private final static ArrayList<String> allowedFileExtension = new ArrayList<>(
           Arrays.asList("djvu", "flac", "gif", "jpg", "jpeg", "mid",
-                  "oga", "ogg","ogv", "opus", "pdf", "png", "svg", "tiff",
+                  "mkv", "oga", "ogg","ogv", "opus", "pdf", "png", "svg", "tiff",
                   "tif", "wav", "webm", "webp", "xcf", "mp3", "stl")
   );
 
@@ -144,7 +150,7 @@ public static ColumnConstraints newColumn(int value, String unit, HPos position)
   );
 
   public static boolean stringHasValidFileExtension(String string) {
-    return allowedExtentionImage.parallelStream().anyMatch(string::endsWith);
+    return allowedFileExtension.parallelStream().anyMatch(string::endsWith);
   }
 
   public static boolean isPossibleBadFilename(String name) {
@@ -207,13 +213,46 @@ public static Map<String, Integer> getFilesByExtention(File[] files) {
     return map;
   }
 
+  // @TODO: remove fancy chars
   public static String getNormalizedName(String name) {
     return name.trim().replaceAll(" +", " ");
   }
 
   public static boolean isFileAllowedToUpload(String name) {
-    return allowedExtentionImage.indexOf(getExtFromFilename(name)) > -1;
+    return allowedFileExtension.indexOf(getExtFromFilename(name)) > -1;
   }
+
+  // @source: https://howtodoinjava.com/java/io/how-to-generate-sha-or-md5-file-checksum-hash-in-java/
+  public static String getFileChecksum(MessageDigest digest, File file) throws IOException {
+    //Get file input stream for reading the file content
+    FileInputStream fis = new FileInputStream(file);
+
+    //Create byte array to read data in chunks
+    byte[] byteArray = new byte[1024];
+    int bytesCount = 0;
+
+    //Read file data and update in message digest
+    while ((bytesCount = fis.read(byteArray)) != -1) {
+        digest.update(byteArray, 0, bytesCount);
+    };
+
+    //close the stream; We don't need it now.
+    fis.close();
+
+    //Get the hash's bytes
+    byte[] bytes = digest.digest();
+
+    //This bytes[] has bytes in decimal format;
+    //Convert it to hexadecimal format
+    StringBuilder sb = new StringBuilder();
+    for(int i=0; i< bytes.length ;i++)
+    {
+        sb.append(Integer.toString((bytes[i] & 0xff) + 0x100, 16).substring(1));
+    }
+
+    //return complete hash
+   return sb.toString();
+}
 
   public static String readUrl(String urlString) throws Exception {
     BufferedReader reader = null;

src/pattypan/panes/ChooseColumnsPane.java

@@ -142,7 +142,7 @@ private WikiPane setContent() {
     wikicodeLink = new Hyperlink(Util.text("choose-columns-wikicode"));
     rightContainer.getChildren().addAll(
             new Region(),
-            new WikiLabel("Advanced").setClass("bold"),
+            new WikiLabel("choose-columns-advanced").setClass("bold"),
             wikicodeLink
     );
 

src/pattypan/panes/UploadPane.java

@@ -40,6 +40,7 @@
 import javafx.scene.layout.VBox;
 import javafx.stage.Stage;
 import javax.security.auth.login.LoginException;
+import org.wikipedia.Wiki;
 import pattypan.Session;
 import pattypan.Settings;
 import pattypan.UploadElement;
@@ -123,6 +124,11 @@ public void changed(ObservableValue<? extends String> ov, String oldValue, Strin
           WikiLabel label = new WikiLabel(text).setAlign("left");
           addInfo(label);
           addInfo(new WikiLabel(""));
+        } else if (value.contains("FILE_DUPLICATE")) {
+          String text = String.format("[%s/%s] ", data[1], max) + Util.text("upload-log-error", Util.text("upload-log-error-file-duplicate", data[3]));
+          WikiLabel label = new WikiLabel(text).setAlign("left");
+          addInfo(label);
+          addInfo(new WikiLabel(""));
         } else if (value.contains("UPLOAD_SUCCESS")) {
           String text = String.format("[%s/%s] ", data[1], max) + Util.text("upload-log-success");
           WikiLabel label = new WikiLabel(text).setAlign("left");
@@ -234,6 +240,29 @@ private boolean isFileNameTaken(String name) {
       return false;
     }
   }
+
+  private String isFileUploaded(UploadElement ue) {
+    String checksum = ue.getFileChecksum();
+    try {
+      Wiki.LogEntry[] entries = Session.WIKI.getChecksumDuplicates(checksum);
+      if(entries.length > 0) {
+        return entries[0].getTarget();
+      }
+      return null;
+    } catch (UnknownHostException ex) {
+      Session.LOGGER.log(Level.WARNING,
+              "Error occurred during file SHA1 check: {0}",
+              new String[]{"no internet connection"}
+      );
+      return null;
+    } catch (IOException ex) {
+      Session.LOGGER.log(Level.WARNING,
+              "Error occurred during file SHA1 check: {0}",
+              new String[]{ex.getLocalizedMessage()}
+      );
+      return null;
+    }
+  }
 
   private void uploadFiles(ArrayList<UploadElement> fileList) {
     Task task = new Task() {
@@ -261,6 +290,17 @@ protected Object call() {
                 skipped++;
                 continue;
               }
+
+              String duplicate = isFileUploaded(ue);
+              if (duplicate != null) {
+                updateMessage(String.format(
+                        "FILE_DUPLICATE | %s | %s | %s",
+                        current + 1, name, duplicate
+                ));
+                Thread.sleep(500);
+                skipped++;
+                continue;
+              }
 
               if (ue.getData("path").startsWith("https://") || ue.getData("path").startsWith("http://")) {
                 Session.WIKI.upload(ue.getUrl(), name, ue.getWikicode(), summary);

src/pattypan/text/messages.properties

@@ -54,6 +54,7 @@ upload-log-canceled=Upload cancel requested. No more images will be uploaded aft
 upload-log-done=Upload completed. %s files uploaded, %s files skipped.
 upload-log-error=Error: %s.
 upload-log-error-name-taken=This file name is already used
+upload-log-error-file-duplicate=This file is already uploaded as %s
 upload-log-success=File uploaded!
 upload-log-uploading=Uploading %s
 upload-name=Upload