From 95925d8f04782c96fc32fe28606d9a9e14d434bf Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 2 Nov 2023 17:31:41 -0700 Subject: [PATCH 1/9] Create generic fflate wrapper utility. --- packages/hashi/package.json | 1 - packages/hashi/src/H5P/H5PRunner.js | 34 ++++------------------- packages/hashi/src/H5P/loadBinary.js | 2 +- packages/kolibri-zip/package.json | 11 ++++++++ packages/kolibri-zip/src/index.js | 41 ++++++++++++++++++++++++++++ yarn.lock | 7 ++++- 6 files changed, 64 insertions(+), 32 deletions(-) create mode 100644 packages/kolibri-zip/package.json create mode 100644 packages/kolibri-zip/src/index.js diff --git a/packages/hashi/package.json b/packages/hashi/package.json index e8087be312..cc368edb8b 100644 --- a/packages/hashi/package.json +++ b/packages/hashi/package.json @@ -24,7 +24,6 @@ "dependencies": { "core-js": "3.33", "dayjs": "^1.11.10", - "fflate": "^0.8.1", "iri": "^1.3.1", "is-language-code": "^3.1.0", "iso8601-duration": "^2.1.1", diff --git a/packages/hashi/src/H5P/H5PRunner.js b/packages/hashi/src/H5P/H5PRunner.js index 43d554ac27..b051c18b3f 100644 --- a/packages/hashi/src/H5P/H5PRunner.js +++ b/packages/hashi/src/H5P/H5PRunner.js @@ -5,7 +5,7 @@ import set from 'lodash/set'; import debounce from 'lodash/debounce'; import unset from 'lodash/unset'; import Toposort from 'toposort-class'; -import { unzip, strFromU8 } from 'fflate'; +import ZipFile from 'kolibri-zip'; import filenameObj from '../../h5p_build.json'; import mimetypes from '../mimetypes.json'; import { XAPIVerbMap } from '../xAPI/xAPIVocabulary'; @@ -13,30 +13,6 @@ import loadBinary from './loadBinary'; const H5PFilename = filenameObj.filename; -class Zip { - constructor(file) { - this.zipfile = file; - } - - _getFiles(filter) { - return new Promise((resolve, reject) => { - unzip(this.zipfile, { filter }, (err, unzipped) => { - if (err) { - reject(err); - } - resolve(Object.entries(unzipped).map(([name, obj]) => ({ name, obj }))); - }); - }); - } - - file(filename) { - return this._getFiles(file => file.name === filename).then(files => files[0]); - } - files(path) { - return this._getFiles(file => file.name.startsWith(path)); - } -} - const CONTENT_ID = '1234567890'; // Verbs that we simply will not report on. @@ -209,7 +185,7 @@ export default class H5PRunner { loadBinary(this.filepath) .then(file => { // Store the zip locally for later reference - this.zip = new Zip(file); + this.zip = new ZipFile(file); // Recurse all the package dependencies return this.recurseDependencies('h5p.json', true); }) @@ -547,7 +523,7 @@ export default class H5PRunner { if (!file) { return; } - const json = JSON.parse(strFromU8(file.obj)); + const json = JSON.parse(file.toString()); const preloadedDependencies = json['preloadedDependencies'] || []; // Make a copy so that we are not modifying the same object visitedPaths = { @@ -643,7 +619,7 @@ export default class H5PRunner { if (fileName === 'content.json') { // Store this special file contents here as raw text // as that is how H5P expects it. - this.contentJson = strFromU8(file.obj); + this.contentJson = file.toString(); } else { // Create blob urls for every item in the content folder this.contentPaths[fileName] = createBlobUrl(file.obj, fileName); @@ -676,7 +652,7 @@ export default class H5PRunner { // If it's a CSS file load as a string from the zipfile for later // replacement of URLs. // For JS or CSS, we load as string to concatenate and later turn into a single file. - this.packageFiles[packagePath][fileName] = strFromU8(file.obj); + this.packageFiles[packagePath][fileName] = file.toString(); } else { // Otherwise just create a blob URL for this file and store it in our packageFiles maps. this.packageFiles[packagePath][fileName] = createBlobUrl(file.obj, fileName); diff --git a/packages/hashi/src/H5P/loadBinary.js b/packages/hashi/src/H5P/loadBinary.js index 9329f17b61..6d782c2e32 100644 --- a/packages/hashi/src/H5P/loadBinary.js +++ b/packages/hashi/src/H5P/loadBinary.js @@ -20,7 +20,7 @@ export default function(path) { if (xhr.readyState === 4) { if (xhr.status === 200 || xhr.status === 0) { try { - resolve(new Uint8Array(xhr.response)); + resolve(xhr.response); } catch (err) { reject(new Error(err)); } diff --git a/packages/kolibri-zip/package.json b/packages/kolibri-zip/package.json new file mode 100644 index 0000000000..62b416c371 --- /dev/null +++ b/packages/kolibri-zip/package.json @@ -0,0 +1,11 @@ +{ + "name": "kolibri-zip", + "version": "0.1.0", + "description": "A library for reading and writing zip files", + "main": "src/index.js", + "author": "Learning Equality", + "license": "MIT", + "dependencies": { + "fflate": "^0.8.1" + } +} \ No newline at end of file diff --git a/packages/kolibri-zip/src/index.js b/packages/kolibri-zip/src/index.js new file mode 100644 index 0000000000..a6d479cdff --- /dev/null +++ b/packages/kolibri-zip/src/index.js @@ -0,0 +1,41 @@ +import { unzip, strFromU8 } from 'fflate'; + +class File { + constructor(name, obj) { + this.name = name; + this.obj = obj; + } + + toString() { + return strFromU8(this.obj); + } +} + +export default class ZipFile { + constructor(file) { + this.zipData = file instanceof Uint8Array ? file : new Uint8Array(file); + } + + _getFiles(filter) { + return new Promise((resolve, reject) => { + unzip(this.zipData, { filter }, (err, unzipped) => { + if (err) { + reject(err); + return; + } + if (!unzipped) { + reject('No files found'); + return; + } + resolve(Object.entries(unzipped).map(([name, obj]) => new File(name, obj))); + }); + }); + } + + file(filename) { + return this._getFiles(file => file.name === filename).then(files => files[0]); + } + files(path) { + return this._getFiles(file => file.name.startsWith(path)); + } +} diff --git a/yarn.lock b/yarn.lock index 83628e7410..0fcf3f8934 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5475,7 +5475,12 @@ fast-levenshtein@^2.0.6, fast-levenshtein@~2.0.6: resolved "https://registry.yarnpkg.com/fast-levenshtein/-/fast-levenshtein-2.0.6.tgz#3d8a5c66883a16a30ca8643e851f19baa7797917" integrity sha1-PYpcZog6FqMMqGQ+hR8Zuqd5eRc= -fastest-levenshtein@^1.0.12, fastest-levenshtein@^1.0.16: +fastest-levenshtein@^1.0.12: + version "1.0.12" + resolved "https://registry.yarnpkg.com/fastest-levenshtein/-/fastest-levenshtein-1.0.12.tgz#9990f7d3a88cc5a9ffd1f1745745251700d497e2" + integrity sha512-On2N+BpYJ15xIC974QNVuYGMOlEVt4s0EOI3wwMqOmK1fdDY+FN/zltPV8vosq4ad4c/gJ1KHScUn/6AWIgiow== + +fastest-levenshtein@^1.0.16: version "1.0.16" resolved "https://registry.yarnpkg.com/fastest-levenshtein/-/fastest-levenshtein-1.0.16.tgz#210e61b6ff181de91ea9b3d1b84fdedd47e034e5" integrity sha512-eRnCtTTtGZFpQCwhJiUOuxPQWRXVKYDn0b2PeHfXL6/Zi53SLAzAHfVhVWK2AryC/WH05kGfxhFIPvTF0SXQzg== From 2bdcd4bfd9bdd62bd43881ad95379850f119f09b Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Sat, 4 Nov 2023 11:28:33 -0700 Subject: [PATCH 2/9] Simplify zip file loading to only need a URL. --- packages/hashi/src/H5P/H5PRunner.js | 73 +++++++++---------- packages/kolibri-zip/src/index.js | 39 ++++++---- .../src/H5P => kolibri-zip/src}/loadBinary.js | 0 3 files changed, 60 insertions(+), 52 deletions(-) rename packages/{hashi/src/H5P => kolibri-zip/src}/loadBinary.js (100%) diff --git a/packages/hashi/src/H5P/H5PRunner.js b/packages/hashi/src/H5P/H5PRunner.js index b051c18b3f..8098585728 100644 --- a/packages/hashi/src/H5P/H5PRunner.js +++ b/packages/hashi/src/H5P/H5PRunner.js @@ -9,7 +9,6 @@ import ZipFile from 'kolibri-zip'; import filenameObj from '../../h5p_build.json'; import mimetypes from '../mimetypes.json'; import { XAPIVerbMap } from '../xAPI/xAPIVocabulary'; -import loadBinary from './loadBinary'; const H5PFilename = filenameObj.filename; @@ -181,45 +180,41 @@ export default class H5PRunner { // and for logging xAPI statements about the content. this.contentNamespace = CONTENT_ID; const start = performance.now(); - // First load the full H5P file as binary so we can read it using JSZip - loadBinary(this.filepath) - .then(file => { - // Store the zip locally for later reference - this.zip = new ZipFile(file); - // Recurse all the package dependencies - return this.recurseDependencies('h5p.json', true); - }) - .then(() => { - // Once we have found all the dependencies, we call this - // to sort the dependencies by their dependencies to make an - // ordered list, with every package being loaded only once its - // dependencies have been loaded. - this.setDependencies(); - return this.processFiles().then(() => { - console.debug(`H5P file processed in ${performance.now() - start} ms`); - this.metadata = pick(this.rootConfig, metadataKeys); - // Do any URL substitition on CSS dependencies - // and turn them into Blob URLs. - // Also order the dendencies according to our sorted - // dependency tree. - this.processCssDependencies(); - this.processJsDependencies(); - // If the iframe has already loaded, start H5P - // Sometimes this check can catch the iframe before it has started - // to load H5P, when it is still blank, but loaded. - // So we also check that H5P is defined on the contentWindow to be sure - // that the ready state applies to the loading of the H5P html file. - if ( - this.iframe.contentDocument && - this.iframe.contentDocument.readyState === 'complete' && - this.iframe.contentWindow.H5P - ) { - return this.initH5P(); - } - // Otherwise wait for the load event. - this.iframe.addEventListener('load', () => this.initH5P()); - }); + // First load the full H5P file + // Store the zip locally for later reference + this.zip = new ZipFile(this.filepath); + // Recurse all the package dependencies + return this.recurseDependencies('h5p.json', true).then(() => { + // Once we have found all the dependencies, we call this + // to sort the dependencies by their dependencies to make an + // ordered list, with every package being loaded only once its + // dependencies have been loaded. + this.setDependencies(); + return this.processFiles().then(() => { + console.debug(`H5P file processed in ${performance.now() - start} ms`); + this.metadata = pick(this.rootConfig, metadataKeys); + // Do any URL substitition on CSS dependencies + // and turn them into Blob URLs. + // Also order the dendencies according to our sorted + // dependency tree. + this.processCssDependencies(); + this.processJsDependencies(); + // If the iframe has already loaded, start H5P + // Sometimes this check can catch the iframe before it has started + // to load H5P, when it is still blank, but loaded. + // So we also check that H5P is defined on the contentWindow to be sure + // that the ready state applies to the loading of the H5P html file. + if ( + this.iframe.contentDocument && + this.iframe.contentDocument.readyState === 'complete' && + this.iframe.contentWindow.H5P + ) { + return this.initH5P(); + } + // Otherwise wait for the load event. + this.iframe.addEventListener('load', () => this.initH5P()); }); + }); } stateUpdated() { diff --git a/packages/kolibri-zip/src/index.js b/packages/kolibri-zip/src/index.js index a6d479cdff..efb08b22a1 100644 --- a/packages/kolibri-zip/src/index.js +++ b/packages/kolibri-zip/src/index.js @@ -1,4 +1,5 @@ import { unzip, strFromU8 } from 'fflate'; +import loadBinary from './loadBinary'; class File { constructor(name, obj) { @@ -12,22 +13,34 @@ class File { } export default class ZipFile { - constructor(file) { - this.zipData = file instanceof Uint8Array ? file : new Uint8Array(file); + constructor(url) { + this._loadingError = null; + this._fileLoadingPromise = loadBinary(url) + .then(data => { + this.zipData = new Uint8Array(data); + }) + .catch(err => { + this._loadingError = err; + }); } _getFiles(filter) { - return new Promise((resolve, reject) => { - unzip(this.zipData, { filter }, (err, unzipped) => { - if (err) { - reject(err); - return; - } - if (!unzipped) { - reject('No files found'); - return; - } - resolve(Object.entries(unzipped).map(([name, obj]) => new File(name, obj))); + if (this._loadingError) { + return Promise.reject(this._loadingError); + } + return this._fileLoadingPromise.then(() => { + return new Promise((resolve, reject) => { + unzip(this.zipData, { filter }, (err, unzipped) => { + if (err) { + reject(err); + return; + } + if (!unzipped) { + reject('No files found'); + return; + } + resolve(Object.entries(unzipped).map(([name, obj]) => new File(name, obj))); + }); }); }); } diff --git a/packages/hashi/src/H5P/loadBinary.js b/packages/kolibri-zip/src/loadBinary.js similarity index 100% rename from packages/hashi/src/H5P/loadBinary.js rename to packages/kolibri-zip/src/loadBinary.js From 00df0d29ce9905642033c25d7d7afcef11cfc9eb Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Sat, 4 Nov 2023 13:30:34 -0700 Subject: [PATCH 3/9] Move blob url creation into kolibri-zip package. --- packages/hashi/package.json | 2 -- packages/hashi/src/H5P/H5PRunner.js | 20 ++----------------- .../generateMimeTypeDB.js} | 0 packages/kolibri-zip/package.json | 6 ++++++ packages/kolibri-zip/src/index.js | 13 ++++++++++++ .../{hashi => kolibri-zip}/src/mimetypes.json | 0 6 files changed, 21 insertions(+), 20 deletions(-) rename packages/{hashi/generateH5PMimeTypeDB.js => kolibri-zip/generateMimeTypeDB.js} (100%) rename packages/{hashi => kolibri-zip}/src/mimetypes.json (100%) diff --git a/packages/hashi/package.json b/packages/hashi/package.json index cc368edb8b..93383a55d6 100644 --- a/packages/hashi/package.json +++ b/packages/hashi/package.json @@ -8,7 +8,6 @@ "build": "yarn run build-base --mode=production", "dev": "yarn run build-base --mode=development --watch", "compat": "eslint -c ./compat.js ./src/*.js", - "mimetypes": "node ./generateH5PMimeTypeDB.js", "build-h5p": "node ./downloadH5PVendor.js && webpack --config ./webpack.config.h5p.js --mode=production" }, "author": "Learning Equality", @@ -17,7 +16,6 @@ "eslint-plugin-compat": "^4.2.0", "html-webpack-plugin": "5.5.3", "jquery": "3.5.1", - "mime-db": "^1.52.0", "mutationobserver-shim": "^0.3.7", "purgecss": "^5.0.0" }, diff --git a/packages/hashi/src/H5P/H5PRunner.js b/packages/hashi/src/H5P/H5PRunner.js index 8098585728..aeef166bed 100644 --- a/packages/hashi/src/H5P/H5PRunner.js +++ b/packages/hashi/src/H5P/H5PRunner.js @@ -7,7 +7,6 @@ import unset from 'lodash/unset'; import Toposort from 'toposort-class'; import ZipFile from 'kolibri-zip'; import filenameObj from '../../h5p_build.json'; -import mimetypes from '../mimetypes.json'; import { XAPIVerbMap } from '../xAPI/xAPIVocabulary'; const H5PFilename = filenameObj.filename; @@ -44,21 +43,6 @@ function contentIdentifier(contentId) { return `cid-${contentId}`; } -/* - * Create a blob and URL for a uint8array - * set the mimetype and return the URL - */ -function createBlobUrl(uint8array, fileName) { - let type = ''; - const fileNameExt = fileName.split('.').slice(-1)[0]; - if (fileNameExt) { - const ext = fileNameExt.toLowerCase(); - type = mimetypes[ext]; - } - const blob = new Blob([uint8array.buffer], { type }); - return URL.createObjectURL(blob); -} - // Looks for any URLs referenced inside url() const cssPathRegex = /(url\(['"]?)([^"')]+)?(['"]?\))/g; @@ -617,7 +601,7 @@ export default class H5PRunner { this.contentJson = file.toString(); } else { // Create blob urls for every item in the content folder - this.contentPaths[fileName] = createBlobUrl(file.obj, fileName); + this.contentPaths[fileName] = file.toUrl(fileName); } } @@ -650,7 +634,7 @@ export default class H5PRunner { this.packageFiles[packagePath][fileName] = file.toString(); } else { // Otherwise just create a blob URL for this file and store it in our packageFiles maps. - this.packageFiles[packagePath][fileName] = createBlobUrl(file.obj, fileName); + this.packageFiles[packagePath][fileName] = file.toUrl(fileName); } } diff --git a/packages/hashi/generateH5PMimeTypeDB.js b/packages/kolibri-zip/generateMimeTypeDB.js similarity index 100% rename from packages/hashi/generateH5PMimeTypeDB.js rename to packages/kolibri-zip/generateMimeTypeDB.js diff --git a/packages/kolibri-zip/package.json b/packages/kolibri-zip/package.json index 62b416c371..38853d0ff0 100644 --- a/packages/kolibri-zip/package.json +++ b/packages/kolibri-zip/package.json @@ -3,9 +3,15 @@ "version": "0.1.0", "description": "A library for reading and writing zip files", "main": "src/index.js", + "scripts": { + "mimetypes": "node ./generateH5PMimeTypeDB.js" + }, "author": "Learning Equality", "license": "MIT", "dependencies": { "fflate": "^0.8.1" + }, + "devDependencies": { + "mime-db": "^1.52.0" } } \ No newline at end of file diff --git a/packages/kolibri-zip/src/index.js b/packages/kolibri-zip/src/index.js index efb08b22a1..9fce65a888 100644 --- a/packages/kolibri-zip/src/index.js +++ b/packages/kolibri-zip/src/index.js @@ -1,5 +1,6 @@ import { unzip, strFromU8 } from 'fflate'; import loadBinary from './loadBinary'; +import mimetypes from './mimetypes.json'; class File { constructor(name, obj) { @@ -10,6 +11,18 @@ class File { toString() { return strFromU8(this.obj); } + + toUrl(fileName = null) { + fileName = fileName || this.name; + let type = ''; + const fileNameExt = fileName.split('.').slice(-1)[0]; + if (fileNameExt) { + const ext = fileNameExt.toLowerCase(); + type = mimetypes[ext]; + } + const blob = new Blob([this.obj.buffer], { type }); + return URL.createObjectURL(blob); + } } export default class ZipFile { diff --git a/packages/hashi/src/mimetypes.json b/packages/kolibri-zip/src/mimetypes.json similarity index 100% rename from packages/hashi/src/mimetypes.json rename to packages/kolibri-zip/src/mimetypes.json From 2efb0258eb5725854d5ea447347453eb2182b1f6 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 14 Nov 2023 13:36:17 -0800 Subject: [PATCH 4/9] Migrate file path substitution logic into kolibri-zip package. --- packages/hashi/src/H5P/H5PRunner.js | 37 +------- packages/hashi/test/H5P.spec.js | 53 ----------- packages/kolibri-zip/src/fileUtils.js | 46 ++++++++++ packages/kolibri-zip/src/index.js | 94 +++++++++++++++---- packages/kolibri-zip/test/fileUtils.spec.js | 99 +++++++++++++++++++++ 5 files changed, 225 insertions(+), 104 deletions(-) delete mode 100644 packages/hashi/test/H5P.spec.js create mode 100644 packages/kolibri-zip/src/fileUtils.js create mode 100644 packages/kolibri-zip/test/fileUtils.spec.js diff --git a/packages/hashi/src/H5P/H5PRunner.js b/packages/hashi/src/H5P/H5PRunner.js index aeef166bed..0f905429f5 100644 --- a/packages/hashi/src/H5P/H5PRunner.js +++ b/packages/hashi/src/H5P/H5PRunner.js @@ -43,33 +43,6 @@ function contentIdentifier(contentId) { return `cid-${contentId}`; } -// Looks for any URLs referenced inside url() -const cssPathRegex = /(url\(['"]?)([^"')]+)?(['"]?\))/g; - -export function replacePaths(dep, packageFiles) { - return packageFiles[dep].replace(cssPathRegex, function(match, p1, p2, p3) { - try { - // Construct a URL with a dummy base so that we can concatenate the - // dependency URL with the URL relative to the dependency - // and then read the pathname to get the new path. - // Take substring to remove the leading slash to match the reference file paths - // in packageFiles. - const path = new URL(p2, new URL(dep, 'http://b.b/')).pathname.substring(1); - // Look to see if there is a URL in our packageFiles mapping that - // that has this as the source path. - const newUrl = packageFiles[path]; - if (newUrl) { - // If so, replace the instance with the new URL. - return `${p1}${newUrl}${p3}`; - } - } catch (e) { - console.debug('Error during URL handling', e); // eslint-disable-line no-console - } - // Otherwise just return the match so that it is unchanged. - return match; - }); -} - const metadataKeys = [ 'title', 'a11yTitle', @@ -580,9 +553,7 @@ export default class H5PRunner { processCssDependencies() { const concatenatedCSS = this.sortedDependencies.reduce((wholeCSS, dependency) => { return (this.cssDependencies[dependency] || []).reduce((allCss, cssDep) => { - const css = replacePaths(cssDep, this.packageFiles[dependency]); - // We have completed the path substition, so concatenate the CSS. - return `${allCss}${css}\n\n`; + return `${allCss}${this.packageFiles[dependency][cssDep]}\n\n`; }, wholeCSS); }, ''); this.cssURL = URL.createObjectURL(new Blob([concatenatedCSS], { type: 'text/css' })); @@ -601,7 +572,7 @@ export default class H5PRunner { this.contentJson = file.toString(); } else { // Create blob urls for every item in the content folder - this.contentPaths[fileName] = file.toUrl(fileName); + this.contentPaths[fileName] = file.toUrl(); } } @@ -634,7 +605,7 @@ export default class H5PRunner { this.packageFiles[packagePath][fileName] = file.toString(); } else { // Otherwise just create a blob URL for this file and store it in our packageFiles maps. - this.packageFiles[packagePath][fileName] = file.toUrl(fileName); + this.packageFiles[packagePath][fileName] = file.toUrl(); } } @@ -647,8 +618,6 @@ export default class H5PRunner { contentFiles.map(file => this.processContent(file)); }), ...Object.keys(this.packageFiles).map(packagePath => { - // JSZip uses regex for path matching, so we first do regex escaping on the packagePath - // in order to get an exact match, and not accidentally do a regex match based on the path return this.zip.files(packagePath).then(packageFiles => { packageFiles.map(file => this.processPackageFile(file, packagePath)); }); diff --git a/packages/hashi/test/H5P.spec.js b/packages/hashi/test/H5P.spec.js deleted file mode 100644 index 07f9d9974a..0000000000 --- a/packages/hashi/test/H5P.spec.js +++ /dev/null @@ -1,53 +0,0 @@ -import { replacePaths } from '../src/H5P/H5PRunner'; - -describe('H5P Path replacement', () => { - describe('CSS path replacement', () => { - it('should replace a simple relative path', () => { - const packageFiles = { - 'package/test.css': 'url("./test.woff")', - 'package/test.woff': 'different', - }; - expect(replacePaths('package/test.css', packageFiles)).toEqual('url("different")'); - }); - it('should replace a more complex relative path', () => { - const packageFiles = { - 'package/css/test.css': 'url("../fonts/test.woff")', - 'package/fonts/test.woff': 'different', - }; - expect(replacePaths('package/css/test.css', packageFiles)).toEqual('url("different")'); - }); - it('should replace paths that use single quotes', () => { - const packageFiles = { - 'package/css/test.css': "url('../fonts/test.woff')", - 'package/fonts/test.woff': 'different', - }; - expect(replacePaths('package/css/test.css', packageFiles)).toEqual("url('different')"); - }); - it('should replace paths that use no quotes', () => { - const packageFiles = { - 'package/css/test.css': 'url(../fonts/test.woff)', - 'package/fonts/test.woff': 'different', - }; - expect(replacePaths('package/css/test.css', packageFiles)).toEqual('url(different)'); - }); - it('should not replace urls that are not a registered path', () => { - const packageFiles = { - 'package/scripts/test.js': 'url(../../../../fonts/test.woff)', - 'package/audio/test.mp3': 'different', - }; - expect(replacePaths('package/scripts/test.js', packageFiles)).toEqual( - 'url(../../../../fonts/test.woff)' - ); - }); - it('should not replace urls that are not a valid path', () => { - // This is mostly to make sure the function does not error. - const packageFiles = { - 'package/scripts/test.js': 'url(flob a dob dib dob)', - 'package/audio/test.mp3': 'different', - }; - expect(replacePaths('package/scripts/test.js', packageFiles)).toEqual( - 'url(flob a dob dib dob)' - ); - }); - }); -}); diff --git a/packages/kolibri-zip/src/fileUtils.js b/packages/kolibri-zip/src/fileUtils.js new file mode 100644 index 0000000000..b0a2a8a2f4 --- /dev/null +++ b/packages/kolibri-zip/src/fileUtils.js @@ -0,0 +1,46 @@ +export function getAbsoluteFilePath(baseFilePath, relativeFilePath) { + // Construct a URL with a dummy base so that we can concatenate the + // dependency URL with the URL relative to the dependency + // and then read the pathname to get the new path. + // Take substring to remove the leading slash to match the reference file paths + // in packageFiles. + try { + return new URL(relativeFilePath, new URL(baseFilePath, 'http://b.b/')).pathname.substring(1); + } catch (e) { + console.debug('Error during URL handling', e); // eslint-disable-line no-console + } + return null; +} + +// Looks for any URLs referenced inside url() +// Handle any query parameters separately. +const cssPathRegex = /(url\(['"]?)([^?"')]+)?(\?[^'"]+)?(['"]?\))/g; + +export function getCSSPaths(fileContents) { + return Array.from(fileContents.matchAll(cssPathRegex), ([, , p2]) => p2); +} + +export function replaceCSSPaths(fileContents, packageFiles) { + return fileContents.replace(cssPathRegex, function(match, p1, p2, p3, p4) { + try { + // Look to see if there is a URL in our packageFiles mapping that + // that has this as the source path. + const newUrl = packageFiles[p2]; + if (newUrl) { + // If so, replace the instance with the new URL. + return `${p1}${newUrl}${p4}`; + } + } catch (e) { + console.debug('Error during URL handling', e); // eslint-disable-line no-console + } + // Otherwise just return the match so that it is unchanged. + return match; + }); +} + +export const defaultFilePathMappers = { + css: { + getPaths: getCSSPaths, + replacePaths: replaceCSSPaths, + }, +}; diff --git a/packages/kolibri-zip/src/index.js b/packages/kolibri-zip/src/index.js index 9fce65a888..0b14854476 100644 --- a/packages/kolibri-zip/src/index.js +++ b/packages/kolibri-zip/src/index.js @@ -1,33 +1,37 @@ -import { unzip, strFromU8 } from 'fflate'; +import { unzip, strFromU8, strToU8 } from 'fflate'; +import isPlainObject from 'lodash/isPlainObject'; import loadBinary from './loadBinary'; import mimetypes from './mimetypes.json'; +import { getAbsoluteFilePath, defaultFilePathMappers } from './fileUtils'; -class File { +class ExtractedFile { constructor(name, obj) { this.name = name; this.obj = obj; } + get fileNameExt() { + return (this.name.split('.').slice(-1)[0] || '').toLowerCase(); + } + + get mimeType() { + return mimetypes[this.fileNameExt] || ''; + } + toString() { return strFromU8(this.obj); } - toUrl(fileName = null) { - fileName = fileName || this.name; - let type = ''; - const fileNameExt = fileName.split('.').slice(-1)[0]; - if (fileNameExt) { - const ext = fileNameExt.toLowerCase(); - type = mimetypes[ext]; - } - const blob = new Blob([this.obj.buffer], { type }); + toUrl() { + const blob = new Blob([this.obj.buffer], { type: this.mimeType }); return URL.createObjectURL(blob); } } export default class ZipFile { - constructor(url) { + constructor(url, { filePathMappers } = { filePathMappers: defaultFilePathMappers }) { this._loadingError = null; + this._extractedFileCache = {}; this._fileLoadingPromise = loadBinary(url) .then(data => { this.zipData = new Uint8Array(data); @@ -35,12 +39,48 @@ export default class ZipFile { .catch(err => { this._loadingError = err; }); + this.filePathMappers = isPlainObject(filePathMappers) ? filePathMappers : {}; } - _getFiles(filter) { - if (this._loadingError) { - return Promise.reject(this._loadingError); + /* + * @param {ExtractedFile} file - The file to carry out replacement of references in + * @param {Object} visitedPaths - A map of paths that have already been visited to prevent a loop + * @return {Promise[ExtractedFile]} - A promise that resolves to the file with references replaced + */ + _replaceFiles(file, visitedPaths) { + const mapper = this.filePathMappers[file.fileNameExt]; + if (!mapper) { + return Promise.resolve(file); } + visitedPaths = { ...visitedPaths }; + visitedPaths[file.name] = true; + const fileContents = file.toString(); + // Filter out any paths that are in our already visited paths, as that means we are in a + // referential loop where one file has pointed us to another, which is now point us back + // to the source. + // Because we need to modify the file before we generate the URL, we can't resolve this loop. + const paths = mapper + .getPaths(fileContents) + .filter(path => !visitedPaths[getAbsoluteFilePath(file.name, path)]); + const absolutePathsMap = paths.reduce((acc, path) => { + acc[getAbsoluteFilePath(file.name, path)] = path; + return acc; + }, {}); + return this._getFiles(file => absolutePathsMap[file.name], visitedPaths).then( + replacementFiles => { + const replacementFileMap = replacementFiles.reduce((acc, replacementFile) => { + acc[absolutePathsMap[replacementFile.name]] = replacementFile.toUrl(); + return acc; + }, {}); + const newFileContents = mapper.replacePaths(fileContents, replacementFileMap); + file.obj = strToU8(newFileContents); + return file; + } + ); + } + + _getFiles(filterPredicate, visitedPaths = {}) { + const filter = file => !this._extractedFileCache[file.name] && filterPredicate(file); return this._fileLoadingPromise.then(() => { return new Promise((resolve, reject) => { unzip(this.zipData, { filter }, (err, unzipped) => { @@ -48,20 +88,40 @@ export default class ZipFile { reject(err); return; } - if (!unzipped) { + const alreadyUnzipped = Object.values(this._extractedFileCache).filter(filterPredicate); + if (!unzipped && !alreadyUnzipped.length) { reject('No files found'); return; } - resolve(Object.entries(unzipped).map(([name, obj]) => new File(name, obj))); + Promise.all( + Object.entries(unzipped).map(([name, obj]) => { + const extractedFile = new ExtractedFile(name, obj); + return this._replaceFiles(extractedFile, visitedPaths).then(extractedFile => { + this._extractedFileCache[name] = extractedFile; + return extractedFile; + }); + }) + ).then(extractedFiles => { + resolve(extractedFiles.concat(alreadyUnzipped)); + }); }); }); }); } file(filename) { + if (this._loadingError) { + return Promise.reject(this._loadingError); + } + if (this._extractedFileCache[filename]) { + return Promise.resolve(this._extractedFileCache[filename]); + } return this._getFiles(file => file.name === filename).then(files => files[0]); } files(path) { + if (this._loadingError) { + return Promise.reject(this._loadingError); + } return this._getFiles(file => file.name.startsWith(path)); } } diff --git a/packages/kolibri-zip/test/fileUtils.spec.js b/packages/kolibri-zip/test/fileUtils.spec.js new file mode 100644 index 0000000000..61622b293d --- /dev/null +++ b/packages/kolibri-zip/test/fileUtils.spec.js @@ -0,0 +1,99 @@ +import { getAbsoluteFilePath, getCSSPaths, replaceCSSPaths } from '../src/fileUtils'; + +describe('File Path replacement', () => { + describe('Absolute path resolution', () => { + it('should resolve a simple relative path', () => { + expect(getAbsoluteFilePath('package/test.css', './test.woff')).toEqual('package/test.woff'); + }); + it('should resolve a more complex relative path', () => { + expect(getAbsoluteFilePath('package/css/test.css', '../fonts/test.woff')).toEqual( + 'package/fonts/test.woff' + ); + }); + }); + describe('CSS path finding', () => { + it('should find a simple relative path', () => { + const packageFiles = ['./test.woff']; + expect(getCSSPaths('url("./test.woff")')).toEqual(packageFiles); + }); + it('should find a more complex relative path', () => { + const packageFiles = ['../fonts/test.woff']; + expect(getCSSPaths('url("../fonts/test.woff")')).toEqual(packageFiles); + }); + it('should find a more complex relative path with query parameters', () => { + const packageFiles = ['../fonts/test.woff']; + expect(getCSSPaths('url("../fonts/test.woff?iefix")')).toEqual(packageFiles); + }); + it('should find paths that use single quotes', () => { + const packageFiles = ['../fonts/test.woff']; + expect(getCSSPaths("url('../fonts/test.woff')")).toEqual(packageFiles); + }); + it('should find paths that use single quotes with query parameters', () => { + const packageFiles = ['../fonts/test.woff']; + expect(getCSSPaths("url('../fonts/test.woff?iefix')")).toEqual(packageFiles); + }); + it('should find paths that use no quotes', () => { + const packageFiles = ['../fonts/test.woff']; + expect(getCSSPaths('url(../fonts/test.woff)')).toEqual(packageFiles); + }); + it('should find paths with no quotes with query parameters', () => { + const packageFiles = ['../fonts/test.woff']; + expect(getCSSPaths('url(../fonts/test.woff?iefix)')).toEqual(packageFiles); + }); + }); + describe('CSS path replacement', () => { + it('should replace a simple relative path', () => { + const packageFiles = { + './test.woff': 'different', + }; + expect(replaceCSSPaths('url("./test.woff")', packageFiles)).toEqual('url("different")'); + }); + it('should replace a more complex relative path', () => { + const packageFiles = { + '../fonts/test.woff': 'different', + }; + expect(replaceCSSPaths('url("../fonts/test.woff")', packageFiles)).toEqual( + 'url("different")' + ); + }); + it('should replace paths that use single quotes', () => { + const packageFiles = { + '../fonts/test.woff': 'different', + }; + expect(replaceCSSPaths("url('../fonts/test.woff')", packageFiles)).toEqual( + "url('different')" + ); + }); + it('should replace paths that use no quotes', () => { + const packageFiles = { + '../fonts/test.woff': 'different', + }; + expect(replaceCSSPaths('url(../fonts/test.woff)', packageFiles)).toEqual('url(different)'); + }); + it('should replace paths that use query parameters', () => { + const packageFiles = { + '../fonts/test.woff': 'different', + }; + expect(replaceCSSPaths('url(../fonts/test.woff?iefix)', packageFiles)).toEqual( + 'url(different)' + ); + }); + it('should not replace urls that are not a registered path', () => { + const packageFiles = { + '../../../../audio/test.mp3': 'different', + }; + expect(replaceCSSPaths('url(../../../../fonts/test.woff)', packageFiles)).toEqual( + 'url(../../../../fonts/test.woff)' + ); + }); + it('should not replace urls that are not a valid path', () => { + // This is mostly to make sure the function does not error. + const packageFiles = { + 'package/audio/test.mp3': 'different', + }; + expect(replaceCSSPaths('url(flob a dob dib dob)', packageFiles)).toEqual( + 'url(flob a dob dib dob)' + ); + }); + }); +}); From d559cc308da4a7dc7841789b345591578fbe849d Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 14 Nov 2023 17:44:50 -0800 Subject: [PATCH 5/9] Extend mimetypes. --- packages/kolibri-zip/generateMimeTypeDB.js | 4 ++++ packages/kolibri-zip/package.json | 2 +- packages/kolibri-zip/src/mimetypes.json | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/kolibri-zip/generateMimeTypeDB.js b/packages/kolibri-zip/generateMimeTypeDB.js index f0df1bf56d..72aa55b537 100644 --- a/packages/kolibri-zip/generateMimeTypeDB.js +++ b/packages/kolibri-zip/generateMimeTypeDB.js @@ -45,6 +45,10 @@ const allowedFileExtensions = new Set([ 'md', 'textile', 'vtt', + // Additional file types for Kolibri + 'html', + 'htm', + 'xhtml', ]); const output = {}; diff --git a/packages/kolibri-zip/package.json b/packages/kolibri-zip/package.json index 38853d0ff0..7645c4739d 100644 --- a/packages/kolibri-zip/package.json +++ b/packages/kolibri-zip/package.json @@ -4,7 +4,7 @@ "description": "A library for reading and writing zip files", "main": "src/index.js", "scripts": { - "mimetypes": "node ./generateH5PMimeTypeDB.js" + "mimetypes": "node ./generateMimeTypeDB.js" }, "author": "Learning Equality", "license": "MIT", diff --git a/packages/kolibri-zip/src/mimetypes.json b/packages/kolibri-zip/src/mimetypes.json index 235f72c1c2..6f78731d3c 100644 --- a/packages/kolibri-zip/src/mimetypes.json +++ b/packages/kolibri-zip/src/mimetypes.json @@ -1 +1 @@ -{"js":"application/javascript","json":"application/json","doc":"application/msword","pdf":"application/pdf","rtf":"text/rtf","xls":"application/vnd.ms-excel","ppt":"application/vnd.ms-powerpoint","odp":"application/vnd.oasis.opendocument.presentation","ods":"application/vnd.oasis.opendocument.spreadsheet","odt":"application/vnd.oasis.opendocument.text","pptx":"application/vnd.openxmlformats-officedocument.presentationml.presentation","xlsx":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet","docx":"application/vnd.openxmlformats-officedocument.wordprocessingml.document","swf":"application/x-shockwave-flash","xml":"text/xml","mp3":"audio/mpeg","m4a":"audio/x-m4a","ogg":"audio/ogg","wav":"audio/x-wav","otf":"font/otf","ttf":"font/ttf","woff":"font/woff","bmp":"image/x-ms-bmp","gif":"image/gif","jpeg":"image/jpeg","jpg":"image/jpeg","png":"image/png","svg":"image/svg+xml","tif":"image/tiff","tiff":"image/tiff","css":"text/css","csv":"text/csv","md":"text/markdown","txt":"text/plain","vtt":"text/vtt","mp4":"video/mp4","webm":"video/webm"} \ No newline at end of file +{"js":"application/javascript","json":"application/json","doc":"application/msword","pdf":"application/pdf","rtf":"text/rtf","xls":"application/vnd.ms-excel","ppt":"application/vnd.ms-powerpoint","odp":"application/vnd.oasis.opendocument.presentation","ods":"application/vnd.oasis.opendocument.spreadsheet","odt":"application/vnd.oasis.opendocument.text","pptx":"application/vnd.openxmlformats-officedocument.presentationml.presentation","xlsx":"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet","docx":"application/vnd.openxmlformats-officedocument.wordprocessingml.document","swf":"application/x-shockwave-flash","xhtml":"application/xhtml+xml","xml":"text/xml","mp3":"audio/mpeg","m4a":"audio/x-m4a","ogg":"audio/ogg","wav":"audio/x-wav","otf":"font/otf","ttf":"font/ttf","woff":"font/woff","bmp":"image/x-ms-bmp","gif":"image/gif","jpeg":"image/jpeg","jpg":"image/jpeg","png":"image/png","svg":"image/svg+xml","tif":"image/tiff","tiff":"image/tiff","css":"text/css","csv":"text/csv","html":"text/html","htm":"text/html","md":"text/markdown","txt":"text/plain","vtt":"text/vtt","mp4":"video/mp4","webm":"video/webm"} \ No newline at end of file From de140435dcd7922fb7ed18d83b60d237d44801fb Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Tue, 14 Nov 2023 17:46:27 -0800 Subject: [PATCH 6/9] Create mapper class. Add support for xml and html file mapping - but only for src attributes. --- packages/kolibri-zip/src/fileUtils.js | 92 +++++++++++++++++++- packages/kolibri-zip/src/index.js | 10 +-- packages/kolibri-zip/test/fileUtils.spec.js | 96 ++++++++++++++++++++- 3 files changed, 188 insertions(+), 10 deletions(-) diff --git a/packages/kolibri-zip/src/fileUtils.js b/packages/kolibri-zip/src/fileUtils.js index b0a2a8a2f4..4d1702c9b0 100644 --- a/packages/kolibri-zip/src/fileUtils.js +++ b/packages/kolibri-zip/src/fileUtils.js @@ -1,3 +1,5 @@ +import flatten from 'lodash/flatten'; + export function getAbsoluteFilePath(baseFilePath, relativeFilePath) { // Construct a URL with a dummy base so that we can concatenate the // dependency URL with the URL relative to the dependency @@ -12,6 +14,20 @@ export function getAbsoluteFilePath(baseFilePath, relativeFilePath) { return null; } +export class Mapper { + constructor(file) { + this.file = file; + } + + getPaths() { + throw new Error('Not implemented'); + } + + replacePaths() { + throw new Error('Not implemented'); + } +} + // Looks for any URLs referenced inside url() // Handle any query parameters separately. const cssPathRegex = /(url\(['"]?)([^?"')]+)?(\?[^'"]+)?(['"]?\))/g; @@ -38,9 +54,77 @@ export function replaceCSSPaths(fileContents, packageFiles) { }); } +class CSSMapper extends Mapper { + getPaths() { + return getCSSPaths(this.file.toString()); + } + + replacePaths(packageFiles) { + return replaceCSSPaths(this.file.toString(), packageFiles); + } +} + +const domParser = new DOMParser(); + +const domSerializer = new XMLSerializer(); + +const attributes = ['src']; + +const attributesSelector = attributes.map(attr => `[${attr}]`).join(', '); + +const queryParamRegex = /([^?)]+)?(\?.*)/g; + +export function getDOMPaths(fileContents, mimeType) { + const dom = domParser.parseFromString(fileContents.trim(), mimeType); + const elements = dom.querySelectorAll(attributesSelector); + return flatten( + Array.from(elements).map(element => + attributes + .map(a => element.getAttribute(a)) + .filter(Boolean) + .map(url => url.replace(queryParamRegex, '$1')) + ) + ); +} + +export function replaceDOMPaths(fileContents, packageFiles, mimeType) { + const dom = domParser.parseFromString(fileContents.trim(), mimeType); + const elements = Array.from(dom.querySelectorAll(attributesSelector)); + for (const element of elements) { + for (const attr of attributes) { + const value = element.getAttribute(attr); + if (!value) { + continue; + } + const newUrl = packageFiles[value.replace(queryParamRegex, '$1')]; + if (newUrl) { + element.setAttribute(attr, newUrl); + } + } + } + if (mimeType === 'text/html') { + // Remove the namespace attribute from the root element + // as serializeToString adds it by default and without this + // it gets repeated. + dom.documentElement.removeAttribute('xmlns'); + } + return domSerializer.serializeToString(dom); +} + +class DOMMapper extends Mapper { + getPaths() { + return getDOMPaths(this.file.toString(), this.file.mimeType); + } + + replacePaths(packageFiles) { + return replaceDOMPaths(this.file.toString(), packageFiles, this.file.mimeType); + } +} + export const defaultFilePathMappers = { - css: { - getPaths: getCSSPaths, - replacePaths: replaceCSSPaths, - }, + css: CSSMapper, + html: DOMMapper, + htm: DOMMapper, + xhtml: DOMMapper, + xml: DOMMapper, }; diff --git a/packages/kolibri-zip/src/index.js b/packages/kolibri-zip/src/index.js index 0b14854476..da13a17ecd 100644 --- a/packages/kolibri-zip/src/index.js +++ b/packages/kolibri-zip/src/index.js @@ -48,19 +48,19 @@ export default class ZipFile { * @return {Promise[ExtractedFile]} - A promise that resolves to the file with references replaced */ _replaceFiles(file, visitedPaths) { - const mapper = this.filePathMappers[file.fileNameExt]; - if (!mapper) { + const mapperClass = this.filePathMappers[file.fileNameExt]; + if (!mapperClass) { return Promise.resolve(file); } visitedPaths = { ...visitedPaths }; visitedPaths[file.name] = true; - const fileContents = file.toString(); + const mapper = new mapperClass(file); // Filter out any paths that are in our already visited paths, as that means we are in a // referential loop where one file has pointed us to another, which is now point us back // to the source. // Because we need to modify the file before we generate the URL, we can't resolve this loop. const paths = mapper - .getPaths(fileContents) + .getPaths() .filter(path => !visitedPaths[getAbsoluteFilePath(file.name, path)]); const absolutePathsMap = paths.reduce((acc, path) => { acc[getAbsoluteFilePath(file.name, path)] = path; @@ -72,7 +72,7 @@ export default class ZipFile { acc[absolutePathsMap[replacementFile.name]] = replacementFile.toUrl(); return acc; }, {}); - const newFileContents = mapper.replacePaths(fileContents, replacementFileMap); + const newFileContents = mapper.replacePaths(replacementFileMap); file.obj = strToU8(newFileContents); return file; } diff --git a/packages/kolibri-zip/test/fileUtils.spec.js b/packages/kolibri-zip/test/fileUtils.spec.js index 61622b293d..3e5750ec23 100644 --- a/packages/kolibri-zip/test/fileUtils.spec.js +++ b/packages/kolibri-zip/test/fileUtils.spec.js @@ -1,4 +1,10 @@ -import { getAbsoluteFilePath, getCSSPaths, replaceCSSPaths } from '../src/fileUtils'; +import { + getAbsoluteFilePath, + getCSSPaths, + replaceCSSPaths, + getDOMPaths, + replaceDOMPaths, +} from '../src/fileUtils'; describe('File Path replacement', () => { describe('Absolute path resolution', () => { @@ -96,4 +102,92 @@ describe('File Path replacement', () => { ); }); }); + const htmlTemplate = src => + ``; + describe('HTML path finding', () => { + const mimeType = 'text/html'; + it('should find a simple relative path', () => { + const packageFiles = ['./test.png']; + expect(getDOMPaths(htmlTemplate('./test.png'), mimeType)).toEqual(packageFiles); + }); + it('should find a more complex relative path', () => { + const packageFiles = ['../fonts/test.png']; + expect(getDOMPaths(htmlTemplate('../fonts/test.png'), mimeType)).toEqual(packageFiles); + }); + it('should find a more complex relative path with query parameters', () => { + const packageFiles = ['../fonts/test.png']; + expect(getDOMPaths(htmlTemplate('../fonts/test.png?iefix'), mimeType)).toEqual(packageFiles); + }); + }); + describe('HTML path replacement', () => { + const mimeType = 'text/html'; + it('should replace a simple relative path', () => { + const packageFiles = { + './test.png': 'different', + }; + expect(replaceDOMPaths(htmlTemplate('./test.png'), packageFiles, mimeType)).toEqual( + htmlTemplate('different') + ); + }); + it('should replace a more complex relative path', () => { + const packageFiles = { + '../fonts/test.png': 'different', + }; + expect(replaceDOMPaths(htmlTemplate('../fonts/test.png'), packageFiles, mimeType)).toEqual( + htmlTemplate('different') + ); + }); + it('should replace paths with query parameters', () => { + const packageFiles = { + '../fonts/test.png': 'different', + }; + expect( + replaceDOMPaths(htmlTemplate('../fonts/test.png?iefix'), packageFiles, mimeType) + ).toEqual(htmlTemplate('different')); + }); + }); + const xmlTemplate = src => + `
`; + describe('XML path finding', () => { + const mimeType = 'text/xml'; + it('should find a simple relative path', () => { + const packageFiles = ['./test.png']; + expect(getDOMPaths(xmlTemplate('./test.png'), mimeType)).toEqual(packageFiles); + }); + it('should find a more complex relative path', () => { + const packageFiles = ['../fonts/test.png']; + expect(getDOMPaths(xmlTemplate('../fonts/test.png'), mimeType)).toEqual(packageFiles); + }); + it('should find a more complex relative path with query parameters', () => { + const packageFiles = ['../fonts/test.png']; + expect(getDOMPaths(xmlTemplate('../fonts/test.png?iefix'), mimeType)).toEqual(packageFiles); + }); + }); + describe('XML path replacement', () => { + const mimeType = 'text/xml'; + it('should replace a simple relative path', () => { + const packageFiles = { + './test.png': 'different', + }; + expect(replaceDOMPaths(xmlTemplate('./test.png'), packageFiles, mimeType)).toEqual( + xmlTemplate('different') + ); + }); + it('should replace a more complex relative path', () => { + const packageFiles = { + '../fonts/test.png': 'different', + }; + expect(replaceDOMPaths(xmlTemplate('../fonts/test.png'), packageFiles, mimeType)).toEqual( + xmlTemplate('different') + ); + }); + it('should replace paths with query parameters', () => { + const packageFiles = { + '../fonts/test.png': 'different', + }; + expect( + replaceDOMPaths(xmlTemplate('../fonts/test.png?iefix'), packageFiles, mimeType) + ).toEqual(xmlTemplate('different')); + }); + }); }); From 9bdec4b4409c274457e979bad7333737cbdedb36 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 16 Nov 2023 15:52:51 -0800 Subject: [PATCH 7/9] Add href substitution to xml and html unzipping. --- packages/kolibri-zip/src/fileUtils.js | 2 +- packages/kolibri-zip/test/fileUtils.spec.js | 60 +++++++++++---------- 2 files changed, 33 insertions(+), 29 deletions(-) diff --git a/packages/kolibri-zip/src/fileUtils.js b/packages/kolibri-zip/src/fileUtils.js index 4d1702c9b0..f1ff0a0d28 100644 --- a/packages/kolibri-zip/src/fileUtils.js +++ b/packages/kolibri-zip/src/fileUtils.js @@ -68,7 +68,7 @@ const domParser = new DOMParser(); const domSerializer = new XMLSerializer(); -const attributes = ['src']; +const attributes = ['src', 'href']; const attributesSelector = attributes.map(attr => `[${attr}]`).join(', '); diff --git a/packages/kolibri-zip/test/fileUtils.spec.js b/packages/kolibri-zip/test/fileUtils.spec.js index 3e5750ec23..a21ed5e75c 100644 --- a/packages/kolibri-zip/test/fileUtils.spec.js +++ b/packages/kolibri-zip/test/fileUtils.spec.js @@ -102,92 +102,96 @@ describe('File Path replacement', () => { ); }); }); - const htmlTemplate = src => - ``; - describe('HTML path finding', () => { + const htmlTemplate = (attr, value) => + ``; + describe.each(['href', 'src'])('HTML path finding for %s', attr => { const mimeType = 'text/html'; it('should find a simple relative path', () => { const packageFiles = ['./test.png']; - expect(getDOMPaths(htmlTemplate('./test.png'), mimeType)).toEqual(packageFiles); + expect(getDOMPaths(htmlTemplate(attr, './test.png'), mimeType)).toEqual(packageFiles); }); it('should find a more complex relative path', () => { const packageFiles = ['../fonts/test.png']; - expect(getDOMPaths(htmlTemplate('../fonts/test.png'), mimeType)).toEqual(packageFiles); + expect(getDOMPaths(htmlTemplate(attr, '../fonts/test.png'), mimeType)).toEqual(packageFiles); }); it('should find a more complex relative path with query parameters', () => { const packageFiles = ['../fonts/test.png']; - expect(getDOMPaths(htmlTemplate('../fonts/test.png?iefix'), mimeType)).toEqual(packageFiles); + expect(getDOMPaths(htmlTemplate(attr, '../fonts/test.png?iefix'), mimeType)).toEqual( + packageFiles + ); }); }); - describe('HTML path replacement', () => { + describe.each(['href', 'src'])('HTML path replacement for %s', attr => { const mimeType = 'text/html'; it('should replace a simple relative path', () => { const packageFiles = { './test.png': 'different', }; - expect(replaceDOMPaths(htmlTemplate('./test.png'), packageFiles, mimeType)).toEqual( - htmlTemplate('different') + expect(replaceDOMPaths(htmlTemplate(attr, './test.png'), packageFiles, mimeType)).toEqual( + htmlTemplate(attr, 'different') ); }); it('should replace a more complex relative path', () => { const packageFiles = { '../fonts/test.png': 'different', }; - expect(replaceDOMPaths(htmlTemplate('../fonts/test.png'), packageFiles, mimeType)).toEqual( - htmlTemplate('different') - ); + expect( + replaceDOMPaths(htmlTemplate(attr, '../fonts/test.png'), packageFiles, mimeType) + ).toEqual(htmlTemplate(attr, 'different')); }); it('should replace paths with query parameters', () => { const packageFiles = { '../fonts/test.png': 'different', }; expect( - replaceDOMPaths(htmlTemplate('../fonts/test.png?iefix'), packageFiles, mimeType) - ).toEqual(htmlTemplate('different')); + replaceDOMPaths(htmlTemplate(attr, '../fonts/test.png?iefix'), packageFiles, mimeType) + ).toEqual(htmlTemplate(attr, 'different')); }); }); - const xmlTemplate = src => - `
`; - describe('XML path finding', () => { + const xmlTemplate = (attr, value) => + `
`; + describe.each(['href', 'src'])('XML path finding for %s', attr => { const mimeType = 'text/xml'; it('should find a simple relative path', () => { const packageFiles = ['./test.png']; - expect(getDOMPaths(xmlTemplate('./test.png'), mimeType)).toEqual(packageFiles); + expect(getDOMPaths(xmlTemplate(attr, './test.png'), mimeType)).toEqual(packageFiles); }); it('should find a more complex relative path', () => { const packageFiles = ['../fonts/test.png']; - expect(getDOMPaths(xmlTemplate('../fonts/test.png'), mimeType)).toEqual(packageFiles); + expect(getDOMPaths(xmlTemplate(attr, '../fonts/test.png'), mimeType)).toEqual(packageFiles); }); it('should find a more complex relative path with query parameters', () => { const packageFiles = ['../fonts/test.png']; - expect(getDOMPaths(xmlTemplate('../fonts/test.png?iefix'), mimeType)).toEqual(packageFiles); + expect(getDOMPaths(xmlTemplate(attr, '../fonts/test.png?iefix'), mimeType)).toEqual( + packageFiles + ); }); }); - describe('XML path replacement', () => { + describe.each(['href', 'src'])('XML path replacement for %s', attr => { const mimeType = 'text/xml'; it('should replace a simple relative path', () => { const packageFiles = { './test.png': 'different', }; - expect(replaceDOMPaths(xmlTemplate('./test.png'), packageFiles, mimeType)).toEqual( - xmlTemplate('different') + expect(replaceDOMPaths(xmlTemplate(attr, './test.png'), packageFiles, mimeType)).toEqual( + xmlTemplate(attr, 'different') ); }); it('should replace a more complex relative path', () => { const packageFiles = { '../fonts/test.png': 'different', }; - expect(replaceDOMPaths(xmlTemplate('../fonts/test.png'), packageFiles, mimeType)).toEqual( - xmlTemplate('different') - ); + expect( + replaceDOMPaths(xmlTemplate(attr, '../fonts/test.png'), packageFiles, mimeType) + ).toEqual(xmlTemplate(attr, 'different')); }); it('should replace paths with query parameters', () => { const packageFiles = { '../fonts/test.png': 'different', }; expect( - replaceDOMPaths(xmlTemplate('../fonts/test.png?iefix'), packageFiles, mimeType) - ).toEqual(xmlTemplate('different')); + replaceDOMPaths(xmlTemplate(attr, '../fonts/test.png?iefix'), packageFiles, mimeType) + ).toEqual(xmlTemplate(attr, 'different')); }); }); }); From 8336bbb4b7b530376281d532bb8a59e93eb02178 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 16 Nov 2023 16:54:19 -0800 Subject: [PATCH 8/9] Add url caching and cleanup behaviour to kolibri-zip utility. --- packages/kolibri-zip/src/index.js | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/kolibri-zip/src/index.js b/packages/kolibri-zip/src/index.js index da13a17ecd..35ae5abec1 100644 --- a/packages/kolibri-zip/src/index.js +++ b/packages/kolibri-zip/src/index.js @@ -8,6 +8,7 @@ class ExtractedFile { constructor(name, obj) { this.name = name; this.obj = obj; + this._url = null; } get fileNameExt() { @@ -23,8 +24,17 @@ class ExtractedFile { } toUrl() { - const blob = new Blob([this.obj.buffer], { type: this.mimeType }); - return URL.createObjectURL(blob); + if (!this._url) { + const blob = new Blob([this.obj.buffer], { type: this.mimeType }); + this._url = URL.createObjectURL(blob); + } + return this._url; + } + + close() { + if (this._url) { + URL.revokeObjectURL(this._url); + } } } @@ -124,4 +134,10 @@ export default class ZipFile { } return this._getFiles(file => file.name.startsWith(path)); } + close() { + for (const file of Object.values(this._extractedFileCache)) { + file.close(); + } + this.zipData = null; + } } From 122d1e90b180e33ec4c640e71c7a75c53519fb38 Mon Sep 17 00:00:00 2001 From: Richard Tibbles Date: Thu, 16 Nov 2023 16:58:11 -0800 Subject: [PATCH 9/9] Update perseus renderer to use kolibri-zip functionality. --- .../assets/src/views/PerseusRendererIndex.vue | 212 +++++++----------- kolibri/plugins/perseus_viewer/package.json | 1 - yarn.lock | 2 +- 3 files changed, 82 insertions(+), 133 deletions(-) diff --git a/kolibri/plugins/perseus_viewer/assets/src/views/PerseusRendererIndex.vue b/kolibri/plugins/perseus_viewer/assets/src/views/PerseusRendererIndex.vue index 58760f0a4c..1d9e954ed1 100644 --- a/kolibri/plugins/perseus_viewer/assets/src/views/PerseusRendererIndex.vue +++ b/kolibri/plugins/perseus_viewer/assets/src/views/PerseusRendererIndex.vue @@ -87,8 +87,8 @@