Changes to support Luis schema v6.0.0 (#534)

* initial changes to suppor 6.0.0 * logic for schema v6.0.0, testing pending * Luis schema v6.0.0 in luisgen c# * Update on test * Update on test * changing import style * changing import style * Fixing build and removing generate tests in LU * Basic test for schema v6.0.0 Co-authored-by: Emilio Munoz <[email protected]>
microsoft · Mar 6, 2020 · 2c061ae · 2c061ae
1 parent 475cc3d
commit 2c061ae
Show file tree

Hide file tree

Showing 25 changed files with 450 additions and 339 deletions.
diff --git a/packages/lu/src/parser/converters/luistocsconverter.js b/packages/lu/src/parser/converters/luistocsconverter.js
@@ -1,6 +1,8 @@
 const parse_multi_platform_luis_1 = require("./../luis/propertyHelper");
 const LuisGenBuilder = require('./../luis/luisGenBuilder')
+const exception = require('./../utils/exception');
 const Writer = require("./helpers/writer");
+const lodash = require("lodash")
 
 module.exports = {
     writeFromLuisJson: async function(luisJson, className, space, outPath) {
@@ -115,7 +117,7 @@ module.exports = {
                     writer.writeLine();
                 }
                 writer.writeLineIndented([
-                    `public class _Instance${composite.compositeName}`,
+                    `public class _Instance${lodash.upperFirst(composite.compositeName)}`,
                     '{'
                 ]);
                 writer.increaseIndentation();
@@ -127,7 +129,7 @@ module.exports = {
                 writer.decreaseIndentation();
                 writer.writeLineIndented([
                     '}',
-                    `public class ${composite.compositeName}Class`,
+                    `public class ${lodash.upperFirst(composite.compositeName)}Class`,
                     '{'
                 ]);
                 writer.increaseIndentation();
@@ -137,12 +139,12 @@ module.exports = {
                 });
                 writer.writeLineIndented([
                     '[JsonProperty("$instance")]',
-                    `public _Instance${composite.compositeName} _instance;`
+                    `public _Instance${lodash.upperFirst(composite.compositeName)} _instance;`
                 ]);
                 writer.decreaseIndentation();
                 writer.writeLineIndented([
                     '}',
-                    `public ${composite.compositeName}Class[] ${composite.compositeName};`
+                    `public ${lodash.upperFirst(composite.compositeName)}Class[] ${composite.compositeName};`
                 ]);
             });
         }
@@ -170,7 +172,18 @@ module.exports = {
             'public _Entities Entities;'
         ]);
     },
-    getEntityWithType: function(entityName, entityType = '') {
+    getEntityWithType: function(entityNameOrObject, entityType = '') {
+        if (typeof entityNameOrObject === 'object' && 'name' in entityNameOrObject){
+            if ('instanceOf' in entityNameOrObject){
+                entityType = entityNameOrObject.instanceOf
+                entityNameOrObject = entityNameOrObject.name
+            } else if (entityNameOrObject.compositeInstanceOf) {
+                let name = parse_multi_platform_luis_1.jsonPropertyName(entityNameOrObject.name)
+                return `public ${lodash.upperFirst(name)}Class[] ${name};`
+            } else {
+                throw (new exception("Invalid LuisGen object: cannot parse entity"))
+            }
+        }
         let result = '';
         switch (entityType) {
             case 'age':
@@ -205,7 +218,7 @@ module.exports = {
             default:
                 result = 'public string[]';
         }
-        return result + ` ${parse_multi_platform_luis_1.jsonPropertyName(entityName)};`;
+        return result + ` ${parse_multi_platform_luis_1.jsonPropertyName(entityNameOrObject)};`;
     },
     converter: function(className, writer) {
         writer.writeLine();

diff --git a/packages/lu/src/parser/luis/luisGenBuilder.js b/packages/lu/src/parser/luis/luisGenBuilder.js
@@ -5,24 +5,52 @@ const retCode = require('./../utils/enums/CLI-errors')
 
 class LuisGenBuilder {
     static build(luisApp) {
-        let result = new LuisGen()
-        try {
-            result.intents = processIntents(luisApp.intents);
-            result.entities = extractEntities(luisApp.entities);
-            result.prebuiltEntities = extractEntities(luisApp.prebuiltEntities, true);
-            result.closedLists = extractEntities(luisApp.closedLists);
-            result.regex_entities = extractEntities(luisApp.regex_entities);
-            result.patternAnyEntities = extractEntities(luisApp.patternAnyEntities);
-            result.composites = extractComposites(luisApp.composites);
-        } catch (err) {
-            throw (new error(retCode.errorCode.INVALID_INPUT_FILE, "Invalid LUIS JSON file content."))
+        let buildWithVersion;
+        if (luisApp.luis_schema_version < "5") {
+            buildWithVersion = buildUpToVersion4;
+        } else if (luisApp.luis_schema_version >= "6.0.0") {
+            buildWithVersion = buildVersion6;
+        } else {
+            throw (new error(retCode.errorCode.INVALID_INPUT_FILE, "Invalid LUIS JSON schema version."))
         }
-        return result
+
+        return buildWithVersion(luisApp);
     }
 }
 
 module.exports = LuisGenBuilder
 
+const buildUpToVersion4 = function(luisApp) {
+    let result = new LuisGen()
+    try {
+        result.intents = processIntents(luisApp.intents);
+        result.entities = extractEntities(luisApp.entities);
+        result.prebuiltEntities = extractEntities(luisApp.prebuiltEntities, true);
+        result.closedLists = extractEntities(luisApp.closedLists);
+        result.regex_entities = extractEntities(luisApp.regex_entities);
+        result.patternAnyEntities = extractEntities(luisApp.patternAnyEntities);
+        result.composites = extractComposites(luisApp.composites);
+    } catch (err) {
+        throw (new error(retCode.errorCode.INVALID_INPUT_FILE, "Invalid LUIS JSON file content."))
+    }
+    return result
+}
+
+const buildVersion6 = function(luisApp) {
+    let result = new LuisGen()
+    try {
+        result.intents = processIntents(luisApp.intents);
+        [result.entities, result.composites] = extractEntitiesV6(luisApp.entities);
+        result.prebuiltEntities = extractEntities(luisApp.prebuiltEntities, true);
+        result.closedLists = extractEntities(luisApp.closedLists);
+        result.regex_entities = extractEntities(luisApp.regex_entities);
+        result.patternAnyEntities = extractEntities(luisApp.patternAnyEntities);
+    } catch (err) {
+        throw (new error(retCode.errorCode.INVALID_INPUT_FILE, "Invalid LUIS JSON file content."))
+    }
+    return result
+}
+
 const processIntents = function(intents) {
     const result = [];
     intents.forEach((intent) => {
@@ -69,4 +97,83 @@ const extractEntities = function(entities, builtIn = false) {
         }
     });
     return result;
+}
+
+const extractEntitiesV6 = function(entities) {
+    // This method provides a simplified topological sort to
+    // solve potential instanceOf dependecies in the v6 entities
+
+    const simpleEntitiesResult = [];
+    const compositeEntitiesResult = [];
+    const simpleEntitiesWithType = {};
+    const resolveEntityType = function(entityName) {
+        const entityStack = [];
+        let entityType = simpleEntitiesWithType[entityName];
+
+        while (simpleEntitiesWithType[entityType]){
+            entityStack.push(entityName);
+            entityName = entityType;
+            entityType = simpleEntitiesWithType[entityName];
+        }
+
+        while (entityName) {
+            simpleEntitiesWithType[entityName] = entityType;
+            entityName = entityStack.pop();
+        }
+    }
+
+    const firstPassStack = entities.slice();
+
+    while(firstPassStack.length) {
+        const entity = firstPassStack.pop();
+
+        if (Array.isArray(entity.children) && entity.children.length) {
+            firstPassStack.push(...entity.children);
+        } else if (!entity.children || (Array.isArray(entity.children) && entity.children.length == 0)) {
+            // is simple entity
+            if (entity.instanceOf) {
+                // If the entity order in the schema was not modified by hand,
+                // this algorithm will solve instanceOf dependencies.
+                const last_type = simpleEntitiesWithType[entity.instanceOf] || entity.instanceOf;
+                simpleEntitiesWithType[entity.name] = last_type;
+            }
+        } else {
+            throw CLIError("Malformed JSON: entity.children should be an array");
+        }
+    }
+
+    // This is a second pass for simple entities.
+    // If the JSON was modified by hand and there's a dependency
+    // in the instanceOf field to an entity that appears later,
+    // the type won't be resolved correctly with one pass.
+    for (const entityName in simpleEntitiesWithType) {
+        resolveEntityType(entityName);
+    }
+
+    const processSimpleEntity = function(entity, listToAppend) {
+        listToAppend.push(
+            entity.instanceOf ? {name: entity.name, instanceOf: simpleEntitiesWithType[entity.instanceOf] || entity.instanceOf} : entity.name
+        )
+    }
+
+    const baseParseEntity = function(entityList, childList, topLevel=false) {
+        entityList.forEach(entity => {
+            if (Array.isArray(entity.children) && entity.children.length) {
+                const compositeEntity = { compositeName: propertyHelper.normalizeName(entity.name), attributes: [] };
+                baseParseEntity(entity.children, compositeEntity.attributes);
+                compositeEntitiesResult.push(compositeEntity);
+                if (!topLevel) {
+                    childList.push({name: entity.name, compositeInstanceOf: true})
+                }
+            } else {
+                processSimpleEntity(
+                    entity, 
+                    topLevel ? simpleEntitiesResult : childList
+                )
+            }
+        });
+    }
+
+    baseParseEntity(entities, null, true);
+    return [simpleEntitiesResult, compositeEntitiesResult];
 }
diff --git a/packages/lu/src/parser/luis/propertyHelper.js b/packages/lu/src/parser/luis/propertyHelper.js
@@ -1,6 +1,9 @@
 
 module.exports = {
     jsonPropertyName: function(property) {
+        if (typeof property === 'object') {
+            property = property.name
+        }
         property+= ''
         let name = property.split(':').slice(-1)[0];
         if (!name.startsWith('geographyV2') &&

diff --git a/packages/lu/test/commands/luis/generate/cs.test.ts b/packages/lu/test/commands/luis/generate/cs.test.ts