direct js import (#91)

* direct js import

* bump version 0.1.45

README.md

@@ -2267,7 +2267,8 @@ export const __init = (templateProcessor) =>{
 }
 ```
 
-The functions can be used in the stated template context
+The functions can be used in the stated template context by using the -xf argument which spreads
+the exported values into the context making them accessible as `$` variables. For example the function `foo()` exported from the js module is available as `$foo`.
 ```json
 > .init -f example/importJS.json --xf=example/test-export.js
 {
@@ -2292,6 +2293,61 @@ This can be combined with the `--importPath` option to import files relative to
   "res": "bar: foo"
 }
 ```
+you can also directly import an entire modules using the `$import` function from within a template.
+You must set the --importPath. For instance, `example/myModule.mjs`contains functions `getGames` and `getPlayers` 
+```js
+export function getGames(){
+    return [
+        "chess",
+        "checkers",
+        "backgammon",
+        "poker",
+        "Theaterwide Biotoxic and Chemical Warfare",
+        "Global Thermonuclear War"
+    ]
+}
+
+export function getPlayers(){
+    return [
+        "dlightman",
+        "prof. Falken",
+        "joshua",
+        "WOPR"
+    ];
+}
+```
+Upon running `example/importLocalJsModule.yaml`, which does `$import('./myModule.mjs')` you will see the field `myModule`
+contains the functions, which are called in the template.
+
+```yaml
+> .init -f example/importLocalJsModule.yaml --importPath=example
+{
+  "myModule": "${$import('./myModule.mjs')}",
+  "games": "${myModule.getGames()}",
+  "players": "${myModule.getPlayers()}"
+}
+> .out
+{
+  "myModule": {
+    "getGames": "{function:}",
+    "getPlayers": "{function:}"
+  },
+  "games": [
+    "chess",
+    "checkers",
+    "backgammon",
+    "poker",
+    "Theaterwide Biotoxic and Chemical Warfare",
+    "Global Thermonuclear War"
+  ],
+  "players": [
+    "dlightman",
+    "prof. Falken",
+    "joshua",
+    "WOPR"
+  ]
+}
+```
 
 #### The __init sidecar function
 If the es module exports a function named `__init`, this function will be invoked with the initialized TemplateProcessor

example/importLocalJsModule.yaml

@@ -0,0 +1,3 @@
+myModule: ${$import('./myModule.mjs')}
+games: ${myModule.getGames()}
+players: ${myModule.getPlayers()}

example/myModule.mjs

@@ -0,0 +1,19 @@
+export function getGames(){
+    return [
+        "chess",
+        "checkers",
+        "backgammon",
+        "poker",
+        "Theaterwide Biotoxic and Chemical Warfare",
+        "Global Thermonuclear War"
+    ]
+}
+
+export function getPlayers(){
+    return [
+        "dlightman",
+        "prof. Falken",
+        "joshua",
+        "WOPR"
+    ];
+}

package.json

@@ -1,6 +1,6 @@
 {
   "name": "stated-js",
-  "version": "0.1.44",
+  "version": "0.1.45",
   "license": "Apache-2.0",
   "description": "JSONata embedded in JSON",
   "main": "./dist/src/index.js",

src/CliCoreBase.ts

@@ -83,7 +83,7 @@ export class CliCoreBase {
         return {...parsed, ...processedArgs}; //spread the processedArgs back into what was parsed
     }
 
-    async readFileAndParse(filepath:string, importPath?:string) {
+    static async readFileAndParse(filepath:string, importPath?:string) {
         const fileExtension = path.extname(filepath).toLowerCase().replace(/\W/g, '');
         if (fileExtension === 'js' || fileExtension === 'mjs') {
             return await import(CliCoreBase.resolveImportPath(filepath, importPath));
@@ -158,7 +158,7 @@ export class CliCoreBase {
             return undefined;
         }
         const input = await this.openFile(filepath);
-        let contextData = contextFilePath ? await this.readFileAndParse(contextFilePath, importPath) : {};
+        let contextData = contextFilePath ? await CliCoreBase.readFileAndParse(contextFilePath, importPath) : {};
         contextData = {...contextData, ...ctx} //--ctx.foo=bar creates ctx={foo:bar}. The dot argument syntax is built into minimist
         options.importPath = importPath; //path is where local imports will be sourced from. We sneak path in with the options
         // if we initialize for the first time, we need to create a new instance of TemplateProcessor
@@ -213,7 +213,7 @@ export class CliCoreBase {
         if(this.currentDirectory){
             _filepath = path.join(this.currentDirectory, _filepath);
         }
-        return await this.readFileAndParse(_filepath);
+        return await CliCoreBase.readFileAndParse(_filepath);
     }
 
 

src/JsonPointer.ts

@@ -245,7 +245,7 @@ export default class JsonPointer {
      */
     static parse(pointer:JsonPointerString) {
         if (pointer === '') { return []; }
-        if (pointer.charAt(0) !== '/') { throw new Error('Invalid JSON pointer: ' + pointer); }
+        if (pointer.charAt(0) !== '/') { throw new Error(`Stated's flavor of JSON pointer Requires JSON Pointers to begin with "/", and this did not: ${pointer}`); }
         return pointer.substring(1).split(/\//).map(JsonPointer.unescape);
     }
 
@@ -265,4 +265,31 @@ export default class JsonPointer {
         const refTokens =  Array.isArray(pointer) ? pointer : JsonPointer.parse(pointer);
         return asArray?refTokens.slice(0,-1):this.compile(refTokens.slice(0,-1));
     }
+
+    /**
+     * Returns true if potentialAncestor is an ancestor of jsonPtr.
+     * For example, if jsonPtr is /a/b/c/d and potentialAncestor is /a/b, this returns true.
+     * @param jsonPtr - The JSON pointer to check.
+     * @param potentialAncestor - The potential ancestor JSON pointer.
+     */
+    static isAncestor(jsonPtr: JsonPointerString, potentialAncestor: JsonPointerString): boolean {
+        // Parse the JSON pointers into arrays of path segments
+        const jsonPtrArray = JsonPointer.parse(jsonPtr);
+        const potentialAncestorArray = JsonPointer.parse(potentialAncestor);
+
+        // If potentialAncestor has more segments than jsonPtr, it cannot be an ancestor
+        if (potentialAncestorArray.length > jsonPtrArray.length) {
+            return false;
+        }
+
+        // Check if each segment in potentialAncestor matches the beginning of jsonPtr
+        for (let i = 0; i < potentialAncestorArray.length; i++) {
+            if (jsonPtrArray[i] !== potentialAncestorArray[i]) {
+                return false;  // If any segment does not match, potentialAncestor is not an ancestor
+            }
+        }
+
+        return true;  // All segments matched, so potentialAncestor is an ancestor of jsonPtr
+    }
+
 }

src/TemplateProcessor.ts

@@ -36,6 +36,7 @@ import {LifecycleOwner, LifecycleState} from "./Lifecycle.js";
 import {LifecycleManager} from "./LifecycleManager.js";
 import {accumulate} from "./utils/accumulate.js";
 import {defaulter} from "./utils/default.js";
+import {CliCoreBase} from "./CliCoreBase.js";
 
 
 declare const BUILD_TARGET: string | undefined;
@@ -652,31 +653,37 @@ export default class TemplateProcessor {
                 this.logger.debug(`Attempting to fetch imported URL '${importMe}'`);
                 resp = await this.fetchFromURL(parsedUrl);
                 resp = this.extractFragmentIfNeeded(resp, parsedUrl);
-            } else if(MetaInfoProducer.EMBEDDED_EXPR_REGEX.test(importMe)){ //this is the case of importing an expression string
+            } else if (MetaInfoProducer.EMBEDDED_EXPR_REGEX.test(importMe)) { //this is the case of importing an expression string
                 resp = importMe; //literally a direction expression like '/${foo}'
-            }else {
-                this.logger.debug(`Attempting local file import of '${importMe}'`);
-                try {
+            } else {
+                this.logger.debug(`Attempting literal import of object as json '${importMe}'`);
+                resp = this.validateAsJSON(importMe);
+                if (resp === undefined) { //it wasn't JSON
+                    this.logger.debug(`Attempting local file import of '${importMe}'`);
+                    const fileExtension = path.extname(importMe).toLowerCase();
                     if (TemplateProcessor._isNodeJS || (typeof BUILD_TARGET !== 'undefined' && BUILD_TARGET !== 'web')) {
-                        resp = await this.localImport(importMe);
+                        try {
+                            resp = await this.localImport(importMe);
+                            if (fileExtension === '.js' || fileExtension === '.mjs') {
+                                return resp; //the module is directly returned and assigned
+                            }
+                        }catch(error){
+                            //we log here and don't rethrow because we don't want to expose serverside path information to remote clients
+                            this.logger.error((error as any).message);
+                        }
+                    } else {
+                        this.logger.error(`It appears we are running in a browser where we can't import from local ${importMe}`)
                     }
-                }catch (error){
-                    this.logger.debug("argument to import doesn't seem to be a file path");
-                }
-
-
-                if(resp === undefined){
-                    this.logger.debug(`Attempting literal import of object '${importMe}'`);
-                    resp = this.validateAsJSON(importMe);
                 }
             }
-            if(resp === undefined){
+            if (resp === undefined) {
                 throw new Error(`Import failed for '${importMe}' at '${metaInfo.jsonPointer__}'`);
             }
             await this.setContentInTemplate(resp, metaInfo);
             return TemplateProcessor.NOOP;
         }
     }
+
     private parseURL(input:string):URL|false {
         try {
             return new URL(input);
@@ -1029,8 +1036,7 @@ export default class TemplateProcessor {
          * @param dependency
          */
         const isCommonPrefix = (exprNode:JsonPointerString, dependency:JsonPointerString):boolean=>{
-            return exprNode.startsWith(dependency) || dependency.startsWith(exprNode);
-
+            return jp.isAncestor(dependency, exprNode);
         }
 
         //metaInfo gets arranged into a tree. The fields that end with "__" are part of the meta info about the
@@ -1716,14 +1722,14 @@ export default class TemplateProcessor {
             return false;
         }
         let existingData;
-        const {sideEffect__=false, value:affectedData} = data || {};
         if (jp.has(output, jsonPtr)) {
             //note get(output, 'foo/-') SHOULD and does return undefined. Don't be tempted into thinking it should
             //return the last element of the array. 'foo/-' syntax only has meaning for update operations. IF we returned
             //the last element of the array, the !isEqual logic below would fail because it would compare the to-be-appended
             //item to what is already there, which is nonsensical.
             existingData = jp.get(output, jsonPtr);
         }
+        const {sideEffect__ = false, value:affectedData} = data || {};
         if (!sideEffect__) {
             if(!isEqual(existingData, data)) {
                 jp.set(output, jsonPtr, data);
@@ -1951,36 +1957,48 @@ export default class TemplateProcessor {
         return null;
     }
 
-    private async localImport(filePathInPackage:string) {
-        // Resolve the package path
+    private async localImport(localPath: string) {
+        this.logger.debug(`importing ${localPath}`);
+        this.logger.debug(`resolving import path using --importPath=${this.options.importPath || ""}`);
+        const fullpath = CliCoreBase.resolveImportPath(localPath, this.options.importPath);
+        this.logger.debug(`resolved import: ${fullpath}`);
         const {importPath} = this.options;
-        let fullPath = filePathInPackage;
-        let content;
-        if (importPath) {
-            // Construct the full file path
-            fullPath = path.join(importPath, filePathInPackage);
-        }
-        try{
-            const fileExtension = path.extname(fullPath).toLowerCase();
+
+        if(!importPath){
+            throw new Error(`$import statements are not allowed in templates unless the importPath is set (see TemplateProcessor.options.importPath and the --importPath command line switch`);
+        }
+
+        // Ensure `fullpath` is within `importPath`. I should be able to $import('./foo.mjs') and$import('./foo.mjs')
+        // but not $import('../../iescaped/foo.mjs)
+        const resolvedImportPath = path.resolve(importPath);
+        if (!fullpath.startsWith(resolvedImportPath)) {
+            throw new Error(`Resolved import path was ${resolvedImportPath} which is outside the allowed --importPath (${importPath})`);
+        }
+
+        try {
+            const fileExtension = path.extname(fullpath).toLowerCase();
+            if (fileExtension === '.js' || fileExtension === '.mjs') {
+                return await import(fullpath);
+            }
+
             // Read the file
-            content =  await fs.promises.readFile(fullPath, 'utf8');
-            if(fileExtension === ".json") {
+            const content = await fs.promises.readFile(fullpath, 'utf8');
+            if (fileExtension === ".json") {
                 return JSON.parse(content);
-            }else if (fileExtension === '.yaml' || fileExtension === '.yml') {
+            } else if (fileExtension === '.yaml' || fileExtension === '.yml') {
                 return yaml.load(content);
-            }else if (fileExtension === '.text' || fileExtension === '.txt') {
+            } else if (fileExtension === '.text' || fileExtension === '.txt') {
                 return content;
-            }else if (fileExtension === '.js' || fileExtension === '.mjs') {
-                throw new Error('js and mjs imports not implemented yet');
-             }else{
-                throw new Error('import file extension must be .json or .yaml or .yml');
+            }else {
+                throw new Error('Import file extension must be .json, .yaml, .yml, .txt, .js, or .mjs');
             }
-        } catch(e) {
+        } catch (e) {
             this.logger.debug('import was not a local file');
             throw e;
         }
     }
 
+
     public static wrapInOrdinaryFunction(jsonataLambda:any) {
         const wrappedFunction = (...args:any[])=> {
             // Call the 'apply' method of jsonataLambda with the captured arguments

src/VizGraph.ts

@@ -98,7 +98,12 @@ export default class VizGraph {
         return dotString;
     }
 
-    static escapeSpecialCharacters(str:string) {
+    static escapeSpecialCharacters(str:string|undefined|null) {
+        // Check if the argument is a valid string
+        if (typeof str !== 'string') {
+            return "--not implemented--";
+        }
+
         // Define the characters to escape and their escaped counterparts
         const specialCharacters = {
             '&': '&',

src/test/TemplateProcessor.test.js

@@ -1302,22 +1302,18 @@ test("local import without --importPath", async () => {
     };
     const tp = new TemplateProcessor(template, {});
     await tp.initialize();
-    expect(tp.output).toEqual({
-        "baz": {
-            "a": 42,
-            "b": 42,
-            "c": "the answer is: 42"
-        },
-        "foo": "bar"
-    });
+    expect(tp.output.baz.error.message).toEqual("Import failed for 'example/ex01.json' at '/baz'");
 });
 
-test("local import with bad filename and no --importPath", async () => {
+test("local import with bad filename", async () => {
     const template = {
         "foo": "bar",
         "baz": "${ $import('example/dingus.json') }"
     };
-    const tp = new TemplateProcessor(template, {});
+    const __filename = fileURLToPath(import.meta.url);
+    const __dirname = dirname(__filename);
+    const importPath = path.join(__dirname, '../', '../');
+    const tp = new TemplateProcessor(template, {}, {importPath});
     await tp.initialize();
     expect(tp.output.baz.error.message).toBe("Import failed for 'example/dingus.json' at '/baz'");
 });

src/utils/GeneratorManager.ts

@@ -21,6 +21,9 @@ export class GeneratorManager{
      * @returns The registered asynchronous generator.
      */
     public generate = (input: AsyncGenerator|any[]|any| (() => any),  options:{valueOnly:boolean, interval?:number, maxYield?:number}={valueOnly:true, interval:-1, maxYield:-1}): AsyncGenerator<any, any, unknown> => {
+        if(input===undefined) {
+            this.templateProcessor.logger.warn("undefined cannot be passed to a generator.");
+        }
         if (this.templateProcessor.isClosed) {
             throw new Error("generate() cannot be called on a closed TemplateProcessor");
         }

src/utils/stringify.ts

@@ -36,6 +36,14 @@ export const circularReplacer = (key: any, value: any) => {
     if (tag === '[object Timeout]'|| (_idleTimeout !== undefined && _onTimeout !== undefined)) { //Node.js
       return "--interval/timeout--";
     }
+    if (tag === '[object Function]') {
+      return "{function:}";
+    }
+    // Check if value is a module-like object
+    // Check if the object has Symbol.toStringTag with value "Module"
+    if (value[Symbol.toStringTag] === '[object Module]') {
+      return "{module:}";
+    }
 
     if (value instanceof Set) {
       return Array.from(value);