Runtime folder structure and default/Resource packaging updates

.gitignore

@@ -20,8 +20,16 @@ npm-debug.log
 # Ignore generated files in output folder
 /dist
 /built
-/runtime
-/src/app-web/static/dlc
+
+# Ignore dataset/templates and runtime artifacts
+/resources
+/data
+templates/*
+# .. but include the blank template
+!templates/_blank/
+
+# Ignore configuration
+.env
 
 # Numerous always-ignore extensions
 *~
@@ -54,7 +62,4 @@ Thumbs.db
 *.zip
 
 # Ignore JetBrains' IDEs
-.idea
-data
-src/system/datasets
-.env
+.idea

.nvmrc

@@ -1 +1 @@
-v10.20.1
+v10.9.0

README-signing.md

@@ -177,7 +177,7 @@ APPLE_TEAM_ID=<Your Apple Team ID>
 
 To Code Sign and Notarize the Electron application:
 1.  `nvm use` -- If you see errors like `node:56749) Warning: Accessing non-existent property 'cat' of module exports...` you're probably using the wrong version of node.
-2.  `npm build package`
+2.  `npm run package`
 3.	`npm run appsign`
 
 This should build and sign an Electron app in `boilerplate/dist/meme-darwin-x64`.  You can distribute the `meme.app` to your teachers
@@ -191,17 +191,16 @@ Test it on other computers to make sure
 
 ### File Locations
 
-`npm run electron` will load the *.js databases in `src/system/datasets/meme/`.
+`npm run electron` will load the *.js databases in `templates/meme/`.
 
-`npm run dev` will load the *.js databases in `src/system/datasets/test/`.
+`npm run dev` will load the *.js databases in `templates/test/`.
 
-When you run the Electron app itself, it'll initialize the database using the data in `src/system/datasets/meme/`.
-The live / changed data is saved in the respective *.loki files in `/runtime`, e.g. `/runtime/meme.loki` and `/runtime/test.loki`.  As are the logs.  These are the files you'll want to grab.
+When you run the Electron app itself, it'll initialize the database using the data in `templates/meme/`.
+The live / changed data is saved in the respective *.loki files in `data/db`, e.g. `data/db/meme.loki` and `data/db/test.loki`.  As are the logs.  These are the files you'll want to grab.
 
-In the code signed and notarized Electron app, the files are in `~/Documents/MEME/db/*.loki`
+(Prior to 2023-08, in the code signed and notarized Electron app, the files are in `~/Documents/MEME/db/*.loki`)
 (Prior to 2023-04, they were stored in `/Contents/Resources/runtime`.)
 
-
 ---
 ---
 

README.md

@@ -41,7 +41,7 @@ This assumes you've already installed NodeJS.  If you haven't, we recommend usin
 
 ### I.B. Download Resources
 
-The pdf and netlogo simulation resources are not currently in the repo.  You will need to download them from the source (ask Joshua for them) and place them in the `boilerplate/src/app-web/static/dlc` folder.
+The pdf and netlogo simulation resources are not currently in the repo.  You will need to download them from the source (ask Joshua for them) and place them in the `resources` folder.
 
 
 ### I.C. Build and Run the Local Server
@@ -63,7 +63,7 @@ This will start the local server and load the existing `meme` database.
 
 You can also create and use arbitrary databases.  E.g. to create a new database, you would:
 
-1. Duplicate `datasets/_blank`, e.g. name it `fall2020`
+1. Duplicate `templates/_blank`, e.g. name it `fall2020`
 1. Start it up using the `DATASET` environment parameter, e.g. `DATASET="fall2020" npm start`
 1. You can quit/reboot/update the server, and start it up again with `DATASET="fall2020" npm start` and the data will be retained.
 
@@ -74,10 +74,10 @@ Once you verify the local server is running, you can build and deploy a standalo
 
 If you want to seed (no pun intended) a MEME Electron application with sample data, e.g. configure teachers, classrooms, and groups, example models, etc, you can just run the MEME application, make the changes, and then duplicate and run the MEME.app.  
 
-The one thing that can't be easily changed via the admin interface are the resources, so generally it's best to download them first.  (Technical note: By running the Electron app, you're automatically loading the `db.js` files in `system/datasets/meme`.  If you want to edit the `db.js` files by hand for the Electron app, edit those.
+The one thing that can't be easily changed via the admin interface are the resources, so generally it's best to download them first.  (Technical note: By running the Electron app, you're automatically seeding the application with the `*.db.js` files from one of the templates stored in `templates`.  If you want to edit the `*.db.js` files by hand for the Electron app, edit those.
 
 To build and run the Electron app:
-1. Make sure all the resources you want to use are in the `boilerplate/src/app-web/static/dlc` folder.
+1. Make sure all the resources you want to use are in one of the MEME templates or in the current resource folder (`/resources`).
 2. `npm run package`
 3. Find the app in `boilerplate/dist/meme-darwin-x64/meme.app`
 4. Double click the `meme.app` file to start it.
@@ -96,16 +96,14 @@ For more technical information about creating and managing the dataset, see:
 
 **Updating Resources in the MEME.app**
 
+TODO: Scott->Ben: Review instructions if there is a better way to describe how to do this in MacOS
+
 If you've already built and distributed the MEME app and find that you need to add or change resources, you can still update resources in the MEME.app manually:
 1. Quit the MEME app.
-2. Find the "meme.app" file in your Finder.
-3. Ctrl-Click on the "meme.app" and select "Show Package Contents"
-4. Navigate to `meme.app/Contents/Resources/app/web/static/dlc`
-5. Copy your new resources into the `dlc` folder.
-6. Run the MEME app and use the admin interface to add the resources and assign them to classrooms.
-7. You can now duplicate the MEME app file and distribute it.  The new resources should be included with the app.
-
-
+2. Find the MEME application folder in your Finder.
+3. Copy your new resources into the `resources` folder.
+4. Run the MEME app and use the admin interface to add the resources and assign them to classrooms.
+5. You can now duplicate the MEME app file and distribute it.  The new resources should be included with the app.
 
 ## III. Admin Interface
 
@@ -208,8 +206,8 @@ To import a database file:
 
 ### Research Logs
 
-* Researcher logs can be found in `meme.app/Contents/Resources/runtime/logs`.  Look for dated log files like `meme.app/Contents/Resources/runtime/logs/2019-09102019-0910-log-102440.txt`
-* Screenshots can be found in `meme.app/Contents/Resources/runtime/screenshots`.
+* Researcher logs can be found in `data/logs`.  Look for dated log files like `data/logs/2019-09102019-0910-log-102440.txt`
+* Screenshots can be found in `data/logs/screenshots`.
 
 NOTE: Over time research logs and screenshots can grow quite large.  You'll want to keep an eye on disk space, especially if you use the same app over months.
 
@@ -225,12 +223,13 @@ NOTE: Over time research logs and screenshots can grow quite large.  You'll want
 We recommend daily backups.  Better yet, back up after each classroom period.
 
 * The easiest way to backup is to just duplicate the whole MEME app.
-* If you want to save space, you can just grab the database file in `meme.app/Contents/Resources/system/datasets/meme.loki` (assuming you didn't rename the database or are running a different database file).
+
+* If you want to save space, you can just grab the database file in `data/db/meme.loki` (assuming you didn't rename the database or are running a different database file).
 
 *Database Snapshots*
 When the server starts up, MEME will now:
 
-* automatically copy the current LOKI database file (usually `meme.loki` in the classroom) to a backup file using the same date format as the log files. The database file is of the form `YYYY-MMDD-meme-HHMMSS.loki.snapshot` and is in the `runtime` directory.
+* automatically copy the current LOKI database file (usually `meme.loki` in the classroom) to a backup file using the same date format as the log files. The database file is of the form `YYYY-MMDD-meme-HHMMSS.loki.snapshot` and is in the `data/db/backups` directory.
 * log the file name of the database snapshot in the log
 
 The snapshot time corresponds to the snapshot log, e.g. `2020-0209-log-124525.loki.snapshot` = state of db at the *start* of the `2020-0209-log-124525.txt` log.

meme.js

@@ -4,7 +4,7 @@ to pass a parameter via npm run script, you have to use -- as in
 npm run myscript -- --myoptions=something
 alternatively you'll just write your own script that does it
 /*/
-const fs = require('fs');
+const fs = require('fs-extra');
 
 if (!fs.existsSync('./node_modules/ip')) {
   console.log(`\x1b[30;41m\x1b[37m MEME STARTUP ERROR \x1b[0m\n`);
@@ -129,9 +129,66 @@ function f_RunElectron() {
   shell.exec(`npx electron ./built/console/console-main`);
 }
 
+function f_GetPlatformConfig(targetPlatform = null, targetArch = null) {
+  // Platform selection; defaults to host platform
+  const electronPlatform = targetPlatform != null ? targetPlatform : process.platform;
+  const electronArch = targetArch != null ? targetArch : process.arch;
+
+  // Note: the Electron Packager does not allow specifying the platform specific folder name
+  //  using the out parameter, only the folder; this variable matches the current package folder
+  //  name
+  const packageFolder = `meme-${electronPlatform}-${electronArch}`;
+  const packageOutput = path.join(__dirname, 'dist', packageFolder);
+
+  let friendlyName;
+  // Note: the entry point is expected to be a subdirectory within the distribution due to conflicts
+  //  between the Electron distribution folder structure and the current MEME folder structure
+  let entrypoint;
+  switch (electronPlatform) {
+    case 'darwin':
+      friendlyName = 'MEME macOS';
+      entrypoint = 'meme.app/Contents/MacOS/meme';
+      break;
+    case 'linux':
+      friendlyName = 'MEME Linux';
+      entrypoint = 'meme.app/meme';
+      break;
+    case 'win32':
+      friendlyName = 'MEME Windows';
+      entrypoint = `meme.app${path.sep}meme`;
+      break;
+    default:
+      console.log(PR, `${CR}ERROR${TR} - unable to identify platform`);
+      process.exit(1);
+  }
+
+  // Defensive check
+  if (!entrypoint.includes(path.sep)) {
+    console.log(
+      PR,
+      'Platform configuration must embed the entry point within a subfolder of ' +
+        'the distribution.'
+    );
+    process.exit(1);
+  }
+
+  const distOutput = path.join(__dirname, 'dist', friendlyName);
+  const appPath = entrypoint.substring(0, entrypoint.indexOf(path.sep));
+
+  return {
+    platform: electronPlatform,
+    arch: electronArch,
+    packageOutput,
+    distOutput,
+    appPath,
+    entrypoint,
+    friendlyName
+  };
+}
+
 function f_PackageApp() {
   console.log(`\n`);
-  console.log(PR, `packaging ${CY}mac electron app${TR} 'meme.app'`);
+  console.log(PR, `packaging ${CY}electron app${TR} 'meme.app'`);
   console.log(PR, `erasing ./built and ./dist directories`);
   shell.rm('-rf', './dist', './built');
   console.log(PR, `compiling console, web, system files into ./built`);
@@ -145,18 +202,76 @@ function f_PackageApp() {
   console.log(PR, `installing node dependencies into ./built`);
   shell.cd('built');
   shell.exec('npm install', { silent: true });
-  console.log(PR, `using electron-packager to write 'meme.app' to ./dist`);
+
+  // FUTURE: Pass the platform and architecture from the script parameters to allow for
+  //  cross-platform builds
+  const platformConfig = f_GetPlatformConfig();
+  const { platform, arch, packageOutput, distOutput, appPath } = f_GetPlatformConfig();
+
+  console.log(PR, `using electron-packager to write '${appPath}' to ${packageOutput}`);
   res = shell.exec(
-    `npx electron-packager . meme --out ../dist --overwrite --app-bundle-id ${APP_BUNDLE_ID}`,
+    `npx electron-packager . meme --platform ${platform} ` +
+      `--arch ${arch} --out ../dist --overwrite ` +
+      `--app-bundle-id ${APP_BUNDLE_ID}`,
     { silent: false }
   );
   // u_checkError(res); // electron-packager stupidly emits status to stderr
-  console.log(PR, `electron app written to ${CY}dist/meme-darwin-x64$/meme.app${TR}`);
-  console.log(PR, `NOTE: default macos security requires ${CR}code signing${TR} to run app.`);
-  console.log(PR, `use ${CY}npm run appsign${TR} to use default developer id (if installed)\n`);
+
+  // Rename the output folder if appropriate
+  if (packageOutput !== distOutput) {
+    console.log(PR, `renaming package folder to friendly platform name: ${distOutput}`);
+    fs.renameSync(packageOutput, distOutput);
+  }
+
+  // For non-Mac platforms, the application is not stored within a dedicated folder and contains
+  //  a 'resources' folder for the Electron resources. Since this conflicts with the 'resources'
+  //  folder for MEME, it should be placed in a dedicated folder
+  if (platform !== 'darwin') {
+    // Move the Electron app and resources into the provided subfolder
+    const tempPath = path.join(__dirname, 'dist', appPath);
+    fs.renameSync(distOutput, tempPath);
+    fs.ensureDirSync(distOutput); // recreate the directory that was just renamed
+    fs.moveSync(tempPath, path.join(distOutput, appPath));
+  }
+
+  console.log(PR, `copying templates to output folder: ${distOutput}`);
+
+  const templatesPath = path.join(__dirname, 'templates');
+  fs.copySync(templatesPath, path.join(distOutput, 'templates'));
+  fs.ensureDirSync(path.join(distOutput, 'data'));
+  fs.ensureDirSync(path.join(distOutput, 'resources'));
+
+  // For macOS - include a shell script to remove the quarantine flag
+  // Note: this is a workaround because the application will run with "translocation" - which will
+  //  application bundle in a read-only folder that is in a randomized path. Runtime file write
+  //  operations therefore fail when they are targetted relative to the application bundle
+  if (platform === 'darwin') {
+    fs.writeFileSync(
+      path.join(distOutput, 'install.sh'),
+      `#/bin/sh\nxattr -d com.apple.quarantine ./${appPath}`,
+      {
+        mode: 0o755
+      }
+    );
+  }
+
+  console.log(PR, `electron app written to ${CY}${distOutput}${TR}`);
+  if (platformConfig.platform === 'darwin') {
+    console.log(PR, `NOTE: default macos security requires ${CR}code signing${TR} to run app.`);
+    console.log(PR, `use ${CY}npm run appsign${TR} to use default developer id (if installed)\n`);
+  }
 }
 
 async function f_SignApp() {
+  const { platform, distOutput, appPath } = f_GetPlatformConfig();
+
+  if (platform !== 'darwin') {
+    console.log(PR, 'Non-MacOS distributions do not require signing (yet).');
+    return;
+  }
+
+  const signedPath = path.join(distOutput, appPath);
+
   console.log(`\n`);
   console.log(PR, `using electron/osx-sign to ${CY}securely sign${TR} 'meme.app'`);
   console.log(PR, `please be patient, this may take a moment...`);
@@ -180,7 +295,7 @@ async function f_SignApp() {
 
   try {
     await signAsync({
-      app: './dist/meme-darwin-x64/meme.app',
+      app: signedPath,
       preAutoEntitlements: false,
       platform: 'darwin',
       optionsForFile: file => {
@@ -201,7 +316,7 @@ async function f_SignApp() {
   try {
     await electronNotarize.notarize({
       appBundleId: APP_BUNDLE_ID,
-      appPath: './dist/meme-darwin-x64/meme.app',
+      appPath: signedPath,
       appleId,
       appleIdPassword,
       teamId,
@@ -221,22 +336,36 @@ async function f_SignApp() {
 }
 
 function f_DebugApp() {
+  // Get the platform config for the current host
+  const { platform, distOutput, appPath, entrypoint } = f_GetPlatformConfig();
+
   console.log(`\n`);
   console.log(PR, `running meme.app with ${CY}console output${TR} to terminal`);
-  console.log(PR, `verifying code signature`);
-  const { code, stderr } = shell.exec('codesign -dvv ./dist/meme-darwin-x64/meme.app', {
-    silent: true
-  });
-  if (code === 0) {
-    console.log(PR, `${CY}console output${TR} from meme.app will appear below`);
-    console.log(PR, `${CY}CTRL-C${TR} or ${CY}close app window${TR} to terminate\n`);
-    shell.exec('./dist/meme-darwin-x64/meme.app/Contents/MacOS/meme');
-  } else {
-    console.log(`\n${stderr.trim()}\n`);
-    console.log(PR, `${CR}ERROR${TR} from codesign check`);
-    console.log(PR, `macos will not run this app until it is signed`);
-    console.log(PR, `if apple developer certs are installed you can run ${CY}npm run appsign${TR}`);
+
+  if (platform === 'darwin') {
+    const signedPath = path.join(distOutput, appPath);
+
+    console.log(PR, `verifying code signature`);
+    const { code, stderr } = shell.exec(`codesign -dvv ${signedPath}`, {
+      silent: true
+    });
+
+    if (code !== 0) {
+      console.log(`\n${stderr.trim()}\n`);
+      console.log(PR, `${CR}ERROR${TR} from codesign check`);
+      console.log(PR, `macos will not run this app until it is signed`);
+      console.log(
+        PR,
+        `if apple developer certs are installed you can run ${CY}npm run appsign${TR}`
+      );
+      return;
+    }
   }
+
+  console.log(PR, `${CY}console output${TR} from meme.app will appear below`);
+  console.log(PR, `${CY}CTRL-C${TR} or ${CY}close app window${TR} to terminate\n`);
+
+  shell.exec(`"${path.join(distOutput, entrypoint)}"`);
 }
 
 function f_DocServe() {
@@ -254,8 +383,8 @@ function f_DocServe() {
 }
 
 function f_Clean(opt) {
-  console.log(PR, `removing dist/, runtime/ and built/ directories...`);
-  shell.rm('-rf', 'dist', 'built', 'runtime');
+  console.log(PR, `removing dist/, data/, and built/ directories...`);
+  shell.rm('-rf', 'dist', 'data', 'built');
   console.log(PR, `directories removed!`);
   if (opt.all) {
     console.log(PR, `also removing node_modules/`);