Desktop: Add OneNote Importer (#11392)

laurent22 · Nov 17, 2024 · 4d7fa59 · 4d7fa59
1 parent e36f377
commit 4d7fa59
Show file tree

Hide file tree

Showing 194 changed files with 15,920 additions and 14 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -85,6 +85,7 @@ plugin_types/
 readme/
 packages/react-native-vosk/lib/
 packages/lib/countable/Countable.js
+packages/onenote-converter/pkg/onenote_converter.js
 
 # AUTO-GENERATED - EXCLUDED TYPESCRIPT BUILD
 packages/app-cli/app/LinkSelector.js
@@ -1181,6 +1182,8 @@ packages/lib/services/interop/InteropService_Importer_Md.test.js
 packages/lib/services/interop/InteropService_Importer_Md.js
 packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.js
 packages/lib/services/interop/InteropService_Importer_Md_frontmatter.js
+packages/lib/services/interop/InteropService_Importer_OneNote.test.js
+packages/lib/services/interop/InteropService_Importer_OneNote.js
 packages/lib/services/interop/InteropService_Importer_Raw.test.js
 packages/lib/services/interop/InteropService_Importer_Raw.js
 packages/lib/services/interop/Module.test.js

diff --git a/.github/scripts/run_ci.sh b/.github/scripts/run_ci.sh
@@ -67,6 +67,7 @@ echo "IS_MACOS=$IS_MACOS"
 echo "Node $( node -v )"
 echo "Npm $( npm -v )"
 echo "Yarn $( yarn -v )"
+echo "Rust $( rustc --version )"
 
 # =============================================================================
 # Install packages

diff --git a/.github/workflows/build-android.yml b/.github/workflows/build-android.yml
@@ -26,6 +26,8 @@ jobs:
           node-version: '18'
           cache: 'yarn'
 
+      - uses: dtolnay/rust-toolchain@stable
+
       - name: Install Yarn
         run: |
           corepack enable

diff --git a/.github/workflows/github-actions-main.yml b/.github/workflows/github-actions-main.yml
@@ -69,6 +69,7 @@ jobs:
 
       - uses: actions/checkout@v4
       - uses: olegtarasov/[email protected]
+      - uses: dtolnay/rust-toolchain@stable
       - uses: actions/setup-node@v4
         with:
           # We need to pin the version to 18.15, because 18.16+ fails with this error:

diff --git a/.gitignore b/.gitignore
@@ -1158,6 +1158,8 @@ packages/lib/services/interop/InteropService_Importer_Md.test.js
 packages/lib/services/interop/InteropService_Importer_Md.js
 packages/lib/services/interop/InteropService_Importer_Md_frontmatter.test.js
 packages/lib/services/interop/InteropService_Importer_Md_frontmatter.js
+packages/lib/services/interop/InteropService_Importer_OneNote.test.js
+packages/lib/services/interop/InteropService_Importer_OneNote.js
 packages/lib/services/interop/InteropService_Importer_Raw.test.js
 packages/lib/services/interop/InteropService_Importer_Raw.js
 packages/lib/services/interop/Module.test.js

diff --git a/Dockerfile.server b/Dockerfile.server
@@ -35,6 +35,9 @@ COPY packages/utils ./packages/utils
 COPY packages/lib ./packages/lib
 COPY packages/server ./packages/server
 
+# We don't want to build onenote-converter since it is not used by the server
+RUN sed --in-place '/onenote-converter/d' ./packages/lib/package.json
+
 # For some reason there's both a .yarn/cache and .yarn/berry/cache that are
 # being generated, and both have the same content. Not clear why it does this
 # but we can delete it anyway. We can delete the cache because we use

diff --git a/joplin.code-workspace b/joplin.code-workspace
@@ -5,6 +5,9 @@
 		},
 	],
 	"settings": {
+		"rust-analyzer.linkedProjects": [
+			"./packages/onenote-converter/Cargo.toml",
+		],
 		"files.exclude": {
 			"_mydocs/mdtest/": true,
 			"_releases/": true,

diff --git a/packages/app-cli/tests/support/onenote/bug_broken_character.zip b/packages/app-cli/tests/support/onenote/bug_broken_character.zip
diff --git a/packages/app-cli/tests/support/onenote/complex_notes.zip b/packages/app-cli/tests/support/onenote/complex_notes.zip
diff --git a/packages/app-cli/tests/support/onenote/group_sections.zip b/packages/app-cli/tests/support/onenote/group_sections.zip
diff --git a/packages/app-cli/tests/support/onenote/many_svgs.html b/packages/app-cli/tests/support/onenote/many_svgs.html
diff --git a/packages/app-cli/tests/support/onenote/simple_notebook.zip b/packages/app-cli/tests/support/onenote/simple_notebook.zip
diff --git a/packages/app-cli/tests/support/onenote/subpages.zip b/packages/app-cli/tests/support/onenote/subpages.zip
diff --git a/packages/app-cli/tests/support/onenote/subsections.zip b/packages/app-cli/tests/support/onenote/subsections.zip
diff --git a/packages/app-cli/tests/support/onenote/svg_with_text_and_style.html b/packages/app-cli/tests/support/onenote/svg_with_text_and_style.html
@@ -0,0 +1,28 @@
+<!DOCTYPE html>
+<html lang="en">
+<body>
+
+<div class="container-outline" style="left: 48px;position: absolute;top: 107px;width: 624px;">
+    <svg viewBox="0 0 240 80" xmlns="http://www.w3.org/2000/svg">
+  <style>
+    .small {
+      font: italic 13px sans-serif;
+    }
+    .heavy {
+      font: bold 30px sans-serif;
+    }
+    /* Note that the color of the text is set with the    *
+     * fill property, the color property is for HTML only */
+    .Rrrrr {
+      font: italic 40px serif;
+      fill: red;
+    }
+  </style>
+  <text x="20" y="35" class="small">My</text>
+  <text x="40" y="35" class="heavy">cat</text>
+  <text x="55" y="55" class="small">is</text>
+  <text x="65" y="55" class="Rrrrr">Grumpy!</text>
+</svg>
+</div>
+</body>
+</html>
diff --git a/packages/app-desktop/gui/Root.tsx b/packages/app-desktop/gui/Root.tsx
@@ -25,6 +25,7 @@ const { ResourceScreen } = require('./ResourceScreen.js');
 import Navigator from './Navigator';
 import WelcomeUtils from '@joplin/lib/WelcomeUtils';
 import JoplinCloudLoginScreen from './JoplinCloudLoginScreen';
+import InteropService from '@joplin/lib/services/interop/InteropService';
 import WindowCommandsAndDialogs from './WindowCommandsAndDialogs/WindowCommandsAndDialogs';
 import { defaultWindowId, stateUtils, WindowState } from '@joplin/lib/reducer';
 import bridge from '../services/bridge';
@@ -91,6 +92,9 @@ async function initialize() {
 		type: 'NOTE_VISIBLE_PANES_SET',
 		panes: Setting.value('noteVisiblePanes'),
 	});
+
+	InteropService.instance().document = document;
+	InteropService.instance().xmlSerializer = new XMLSerializer();
 }
 
 // eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied

diff --git a/packages/lib/BaseModel.ts b/packages/lib/BaseModel.ts
@@ -1,11 +1,11 @@
 import paginationToSql from './models/utils/paginationToSql';
 import Database from './database';
-import uuid from './uuid';
 import time from './time';
 import JoplinDatabase, { TableField } from './JoplinDatabase';
 import { LoadOptions, SaveOptions } from './models/utils/types';
 import ActionLogger, { ItemActionType as ItemActionType } from './utils/ActionLogger';
 import { BaseItemEntity, SqlQuery } from './services/database/types';
+import uuid from './uuid';
 const Mutex = require('async-mutex').Mutex;
 
 // New code should make use of this enum
@@ -80,6 +80,8 @@ class BaseModel {
 		['TYPE_COMMAND', ModelType.Command],
 	];
 
+	private static uuidGenerator: ()=> string = uuid.create;
+
 	public static TYPE_NOTE = ModelType.Note;
 	public static TYPE_FOLDER = ModelType.Folder;
 	public static TYPE_SETTING = ModelType.Setting;
@@ -576,7 +578,7 @@ class BaseModel {
 
 		if (options.isNew) {
 			if (this.useUuid() && !o.id) {
-				modelId = uuid.create();
+				modelId = this.generateUuid();
 				o.id = modelId;
 			}
 
@@ -757,6 +759,15 @@ class BaseModel {
 		return this.db_;
 	}
 
+	public static generateUuid() {
+		return this.uuidGenerator();
+	}
+
+	public static setIdGenerator(generator: ()=> string) {
+		const previous = this.uuidGenerator;
+		this.uuidGenerator = generator;
+		return previous;
+	}
 	// static isReady() {
 	// 	return !!this.db_;
 	// }

diff --git a/packages/lib/package.json b/packages/lib/package.json
@@ -17,9 +17,11 @@
   },
   "devDependencies": {
     "@testing-library/react-hooks": "8.0.1",
+    "@types/adm-zip": "0.5.5",
     "@types/fs-extra": "11.0.4",
     "@types/jest": "29.5.12",
     "@types/js-yaml": "4.0.9",
+    "@types/jsdom": "21.1.6",
     "@types/markdown-it": "13.0.9",
     "@types/mustache": "4.2.5",
     "@types/node": "18.19.42",
@@ -29,6 +31,7 @@
     "canvas": "2.11.2",
     "clean-html": "1.5.0",
     "jest": "29.7.0",
+    "jsdom": "23.2.0",
     "pdfjs-dist": "3.11.174",
     "react": "18.3.1",
     "react-test-renderer": "18.3.1",
@@ -44,11 +47,13 @@
     "@joplin/fork-sax": "^1.2.56",
     "@joplin/fork-uslug": "^1.0.17",
     "@joplin/htmlpack": "~3.2",
+    "@joplin/onenote-converter": "0.0.1",
     "@joplin/renderer": "~3.2",
     "@joplin/turndown": "^4.0.74",
     "@joplin/turndown-plugin-gfm": "^1.0.56",
     "@joplin/utils": "~3.2",
     "@types/nanoid": "3.0.0",
+    "adm-zip": "0.5.12",
     "async-mutex": "0.5.0",
     "base-64": "1.0.0",
     "base64-stream": "1.0.0",

diff --git a/packages/lib/services/interop/InteropService.ts b/packages/lib/services/interop/InteropService.ts
@@ -30,6 +30,8 @@ export default class InteropService {
 	// eslint-disable-next-line @typescript-eslint/no-explicit-any -- Old code before rule was applied
 	private eventEmitter_: any = null;
 	private static instance_: InteropService;
+	private document_: Document;
+	private xmlSerializer_: XMLSerializer;
 
 	public static instance(): InteropService {
 		if (!this.instance_) this.instance_ = new InteropService();
@@ -133,6 +135,14 @@ export default class InteropService {
 					isNoteArchive: false, // Tells whether the file can contain multiple notes (eg. Enex or Jex format)
 					description: _('Text document'),
 				}, () => new InteropService_Importer_Md()),
+
+				makeImportModule({
+					format: 'zip',
+					fileExtensions: ['zip'],
+					sources: [FileSystemItem.File],
+					isNoteArchive: false, // Tells whether the file can contain multiple notes (eg. Enex or Jex format)
+					description: _('OneNote Notebook'),
+				}, dynamicRequireModuleFactory('./InteropService_Importer_OneNote')),
 			];
 
 			const exportModules = [
@@ -189,6 +199,22 @@ export default class InteropService {
 		this.eventEmitter_.emit('modulesChanged');
 	}
 
+	public set xmlSerializer(xmlSerializer: XMLSerializer) {
+		this.xmlSerializer_ = xmlSerializer;
+	}
+
+	public get xmlSerializer() {
+		return this.xmlSerializer_;
+	}
+
+	public set document(document: Document) {
+		this.document_ = document;
+	}
+
+	public get document() {
+		return this.document_;
+	}
+
 	// Find the module that matches the given type ("importer" or "exporter")
 	// and the given format. Some formats can have multiple associated importers
 	// or exporters, such as ENEX. In this case, the one marked as "isDefault"
@@ -273,6 +299,8 @@ export default class InteropService {
 			format: 'auto',
 			destinationFolderId: null,
 			destinationFolder: null,
+			xmlSerializer: this.xmlSerializer,
+			document: this.document,
 			...options,
 		};