Add config option to strip Unicode from entry filenames.

package.json

@@ -129,6 +129,7 @@
   "dependencies": {
     "classnames": "^2.2.5",
     "create-react-class": "^15.6.0",
+    "diacritics": "^1.3.0",
     "fuzzy": "^0.1.1",
     "gotrue-js": "^0.9.15",
     "gray-matter": "^3.0.6",

src/backends/backend.js

@@ -11,6 +11,7 @@ import {
 } from "Reducers/collections";
 import { createEntry } from "ValueObjects/Entry";
 import { sanitizeSlug } from "Lib/urlHelper";
+import diacritics from 'diacritics';
 import TestRepoBackend from "./test-repo/implementation";
 import GitHubBackend from "./github/implementation";
 import GitGatewayBackend from "./git-gateway/implementation";
@@ -41,7 +42,7 @@ class LocalStorageAuthStore {
   }
 }
 
-const slugFormatter = (template = "{{slug}}", entryData) => {
+const slugFormatter = (template = "{{slug}}", entryData, slugType) => {
   const date = new Date();
 
   const getIdentifier = (entryData) => {
@@ -79,7 +80,12 @@ const slugFormatter = (template = "{{slug}}", entryData) => {
   // Replace periods and spaces with dashes.
   .replace(/[.\s]/g, '-');
 
-  return sanitizeSlug(slug);
+  if (slugType === "latin") {
+    const latinSlug = diacritics.remove(slug);
+    return sanitizeSlug(latinSlug, { slugType: "ascii" });
+  }
+
+  return sanitizeSlug(slug, { slugType });
 };
 
 class Backend {
@@ -239,7 +245,7 @@ class Backend {
       if (!selectAllowNewEntries(collection)) {
         throw (new Error("Not allowed to create new entries in this collection"));
       }
-      const slug = slugFormatter(collection.get("slug"), entryDraft.getIn(["entry", "data"]));
+      const slug = slugFormatter(collection.get("slug"), entryDraft.getIn(["entry", "data"]), config.get("slug_type"));
       const path = selectEntryPath(collection, slug);
       entryObj = {
         path,

src/lib/urlHelper.js

@@ -34,27 +34,37 @@ export function stripProtocol(url) {
  */
 const uriChars = /[\w\-.~]/i;
 const ucsChars = /[\xA0-\u{D7FF}\u{F900}-\u{FDCF}\u{FDF0}-\u{FFEF}\u{10000}-\u{1FFFD}\u{20000}-\u{2FFFD}\u{30000}-\u{3FFFD}\u{40000}-\u{4FFFD}\u{50000}-\u{5FFFD}\u{60000}-\u{6FFFD}\u{70000}-\u{7FFFD}\u{80000}-\u{8FFFD}\u{90000}-\u{9FFFD}\u{A0000}-\u{AFFFD}\u{B0000}-\u{BFFFD}\u{C0000}-\u{CFFFD}\u{D0000}-\u{DFFFD}\u{E1000}-\u{EFFFD}]/u;
+const validURIChar = (char) => (uriChars.test(char));
 const validIRIChar = (char) => (uriChars.test(char) || ucsChars.test(char));
-// `sanitizeIRI` does not actually URI-encode the chars (that is the browser's and server's job), just removes the ones that are not allowed.
-export function sanitizeIRI(str, { replacement = "" } = {}) {
+// `sanitizeURI` does not actually URI-encode the chars (that is the browser's and server's job), just removes the ones that are not allowed.
+export function sanitizeURI(str, { replacement = "", type = "iri" } = {}) {
   if (!isString(str)) throw "The input slug must be a string.";
   if (!isString(replacement)) throw "`options.replacement` must be a string.";
+
+  let validChar;
+  if (type === "iri") {
+    validChar = validIRIChar;
+  } else if (type === "ascii") {
+    validChar = validURIChar;
+  } else {
+    throw '`options.type` must be "iri" or "ascii".';
+  }
 
   // Check and make sure the replacement character is actually a safe char itself.
-  if (!Array.from(replacement).every(validIRIChar)) throw "The replacement character(s) (options.replacement) is itself unsafe.";
+  if (!Array.from(replacement).every(validChar)) throw "The replacement character(s) (options.replacement) is itself unsafe.";
 
   // `Array.from` must be used instead of `String.split` because
   //   `split` converts things like emojis into UTF-16 surrogate pairs.
-  return Array.from(str).map(char => (validIRIChar(char) ? char : replacement)).join('');
+  return Array.from(str).map(char => (validChar(char) ? char : replacement)).join('');
 }
 
-export function sanitizeSlug(str, { replacement = '-' } = {}) {
+export function sanitizeSlug(str, { replacement = '-', slugType } = {}) {
   if (!isString(str)) throw "The input slug must be a string.";
   if (!isString(replacement)) throw "`options.replacement` must be a string.";
 
-  // Sanitize as IRI (i18n URI) and as filename.
+  // Sanitize as URI and as filename.
   const sanitize = flow([
-    partialRight(sanitizeIRI, { replacement }),
+    partialRight(sanitizeURI, { replacement, type: slugType }),
     partialRight(sanitizeFilename, { replacement }),
   ]);
   const sanitizedSlug = sanitize(str);

yarn.lock

@@ -2302,6 +2302,10 @@ detect-node@^2.0.3:
   version "2.0.3"
   resolved "https://registry.yarnpkg.com/detect-node/-/detect-node-2.0.3.tgz#a2033c09cc8e158d37748fbde7507832bd6ce127"
 
+diacritics@^1.3.0:
+  version "1.3.0"
+  resolved "https://registry.yarnpkg.com/diacritics/-/diacritics-1.3.0.tgz#3efa87323ebb863e6696cebb0082d48ff3d6f7a1"
+
 diff@^3.2.0:
   version "3.4.0"
   resolved "https://registry.yarnpkg.com/diff/-/diff-3.4.0.tgz#b1d85507daf3964828de54b37d0d73ba67dda56c"