[Ludown] fix validate luis json api to not modify source json structure (#1254)

* Fix for 1230

* Fix for #1232

* Fixing tests that fail on windows

* Fix broken tests

* Updating tests

packages/Ludown/lib/parseFileContents.js

@@ -31,17 +31,7 @@ const parseFileContentsModule = {
      * @throws {exception} Throws on errors. exception object includes errCode and text. 
      */
     validateLUISBlob: async function (LUISJSONBlob) {
-        // patterns can have references to any other entity types. 
+        
-        // So if there is a pattern.any entity that is also defined as another type, remove the pattern.any entity
-        LUISJSONBlob.patternAnyEntities = (LUISJSONBlob.patternAnyEntities || []).filter(entity => {
-            if (itemExists(LUISJSONBlob.entities, entity.name, entity.roles)) return false;
-            if (itemExists(LUISJSONBlob.closedLists, entity.name, entity.roles)) return false;
-            if (itemExists(LUISJSONBlob.model_features, entity.name, entity.roles)) return false;
-            if (itemExists(LUISJSONBlob.prebuiltEntities, entity.name, entity.roles)) return false;
-            if (itemExists(LUISJSONBlob.regex_entities, entity.name, entity.roles)) return false;
-            if (itemExists(LUISJSONBlob.prebuiltEntities, entity.name, entity.roles)) return false;
-            return true;
-        });
 
         // look for entity name collisions - list, simple, patternAny, phraselist
         // look for list entities labelled
@@ -380,6 +370,40 @@ const parseFileContentsModule = {
                     }
                 }
             });
+
+            // do we have pattern.any entities here? 
+            (blob.patternAnyEntities || []).forEach(patternAny => {
+                let paIdx = -1;
+                let patternAnyInMaster = FinalLUISJSON.patternAnyEntities.find((item, idx) => {
+                    if (item.name === patternAny.name) {
+                        paIdx = idx;
+                        return true;
+                    }
+                    return false;
+                });
+                // verify that this patternAny entity does not exist as any other type
+                let simpleEntityInMaster = FinalLUISJSON.entities.find(item => item.name == patternAny.name);
+                let compositeInMaster = FinalLUISJSON.composites.find(item => item.name == patternAny.name);
+                let listEntityInMaster = FinalLUISJSON.closedLists.find(item => item.name == patternAny.name);
+                let regexEntityInMaster = FinalLUISJSON.regex_entities.find(item => item.name == patternAny.name);
+                let prebuiltInMaster = FinalLUISJSON.prebuiltEntities.find(item => item.name == patternAny.name);
+                if (!simpleEntityInMaster && 
+                    !compositeInMaster &&
+                    !listEntityInMaster &&
+                    !regexEntityInMaster &&
+                    !prebuiltInMaster) {
+                    if (patternAnyInMaster) {
+                        (patternAny.roles || []).forEach(role => !patternAnyInMaster.roles.includes(role) ? patternAnyInMaster.roles.push(role) : undefined);
+                    } else {
+                            FinalLUISJSON.patternAnyEntities.push(patternAny);
+                    }
+                } else {
+                    // remove the pattern.any from master if another entity type has this name.
+                    if (patternAnyInMaster) {
+                        if (paIdx !== -1) FinalLUISJSON.patternAnyEntities.splice(paIdx, 1);
+                    }
+                }
+            })
         });
         return FinalLUISJSON;
     },
@@ -645,16 +669,22 @@ const parseAndHandleEntity = function (parsedContent, chunkSplitByLine, locale,
     let pEntityName = (entityName === 'PREBUILT') ? entityType : entityName;
     // see if we already have this as Pattern.Any entity
     // see if we already have this in patternAny entity collection; if so, remove it but remember the roles (if any)
-    for (let i in parsedContent.LUISJsonStructure.patternAnyEntities) {
+    let patternAnyMatchIdx = new Array();
-        if (parsedContent.LUISJsonStructure.patternAnyEntities[i].name === pEntityName) {
+    (parsedContent.LUISJsonStructure.patternAnyEntities || []).forEach((entity, idx) => {
+        if (entity.name === pEntityName || entity.name === entityType) {
             if (entityType.toLowerCase().trim().includes('phraselist')) {
                 throw (new exception(retCode.errorCode.INVALID_INPUT, '[ERROR]: Phrase lists cannot be used as an entity in a pattern "' + pEntityName));
             }
-            if (parsedContent.LUISJsonStructure.patternAnyEntities[i].roles.length !== 0) entityRoles = parsedContent.LUISJsonStructure.patternAnyEntities[i].roles;
+            if (entity.roles.length !== 0) entityRoles = entity.roles;
-            parsedContent.LUISJsonStructure.patternAnyEntities.splice(i, 1);
+            patternAnyMatchIdx.push(idx);
-            break;
         }
-    }
+    });
+
+    // remove any found patternany entities.
+    patternAnyMatchIdx.sort().reverse().forEach(idx => {
+        parsedContent.LUISJsonStructure.patternAnyEntities.splice(idx, 1);
+    })
+
     // add this entity to appropriate place
     // is this a builtin type? 
     if (builtInTypes.consolidatedList.includes(entityType)) {
@@ -1076,13 +1106,24 @@ const parseAndHandleIntent = function (parsedContent, chunkSplitByLine) {
                     if (!parsedContent.LUISJsonStructure.patterns.find(item => deepEqual(item, newPattern))) {
                         parsedContent.LUISJsonStructure.patterns.push(newPattern);
                     } 
-                    // add all entities to pattern.Any.
+                    // add all entities to pattern.Any only if they do not have another type.
                     entitiesFound.forEach(entity => {
-                        if (entity.role !== '') {
+                        let simpleEntityInMaster = parsedContent.LUISJsonStructure.entities.find(item => item.name == entity.entity);
-                            addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity.entity, [entity.role.trim()])
+                        let compositeInMaster = parsedContent.LUISJsonStructure.composites.find(item => item.name == entity.entity);
-                        } else {
+                        let listEntityInMaster = parsedContent.LUISJsonStructure.closedLists.find(item => item.name == entity.entity);
-                            addItemIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity.entity);
+                        let regexEntityInMaster = parsedContent.LUISJsonStructure.regex_entities.find(item => item.name == entity.entity);
-                        }
+                        let prebuiltInMaster = parsedContent.LUISJsonStructure.prebuiltEntities.find(item => item.name == entity.entity);
+                        if (!simpleEntityInMaster && 
+                            !compositeInMaster &&
+                            !listEntityInMaster &&
+                            !regexEntityInMaster &&
+                            !prebuiltInMaster) {
+                                if (entity.role !== '') {
+                                    addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity.entity, [entity.role.trim()])
+                                } else {
+                                    addItemIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity.entity);
+                                }                            
+                        }                         
                     });
 
                     // if this intent does not have any utterances, push this pattern as an utterance as well. 

packages/Ludown/lib/parser.js

@@ -418,13 +418,50 @@ const resolveReferencesInUtterances = async function(allParsedContent) {
 
                 entitiesFound.forEach(function (entity) {
                     entity = entity.replace("{", "").replace("}", "");
-
+                    let entityName = entity;
+                    let roleName = '';
                     if (entity.includes(':')) {
                         // this is an entity with role
-                        const [entityName, roleName] = entity.split(':');
+                        [entityName, roleName] = entity.split(':');
-                        parseFileContents.addItemOrRoleIfNotPresent(luisModel.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entityName, [roleName])
+                    }
+                    // insert the entity only if it does not already exist
+                    let simpleEntityInMaster = luisModel.LUISJsonStructure.entities.find(item => item.name == entityName);
+                    let compositeInMaster = luisModel.LUISJsonStructure.composites.find(item => item.name == entityName);
+                    let listEntityInMaster = luisModel.LUISJsonStructure.closedLists.find(item => item.name == entityName);
+                    let regexEntityInMaster = luisModel.LUISJsonStructure.regex_entities.find(item => item.name == entityName);
+                    let prebuiltInMaster = luisModel.LUISJsonStructure.prebuiltEntities.find(item => item.name == entityName);
+                    let paIdx = -1;
+                    let patternAnyInMaster = luisModel.LUISJsonStructure.patternAnyEntities.find((item, idx) => {
+                        if (item.name === entityName) {
+                            paIdx = idx;
+                            return true;
+                        }
+                        return false;
+                    });
+                    if (!simpleEntityInMaster && 
+                        !compositeInMaster &&
+                        !listEntityInMaster &&
+                        !regexEntityInMaster &&
+                        !prebuiltInMaster) {
+                            if (!patternAnyInMaster) {
+                                // add a pattern.any entity
+                                if (roleName !== '') {
+                                    parseFileContents.addItemOrRoleIfNotPresent(luisModel.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entityName, [roleName])
+                                } else {
+                                    parseFileContents.addItemIfNotPresent(luisModel.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity);
+                                }
+                            } else {
+                                // add the role if it does not exist already.
+                                if (roleName !== '') {
+                                    !patternAnyInMaster.roles.includes(roleName) ? patternAnyInMaster.roles.push(roleName) : undefined;
+                                }
+                            }
                     } else {
-                        parseFileContents.addItemIfNotPresent(luisModel.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity);
+                        // we found this pattern.any entity as another type.
+                        if (patternAnyInMaster && paIdx !== -1) {
+                            // remove the patternAny entity from the list because it has been explicitly defined elsewhere.
+                            luisModel.LUISJsonStructure.patternAnyEntities.splice(paIdx, 1);
+                        }
                     }
                 });
             }

packages/Ludown/test/ludown.examples.test.suite.js

@@ -601,7 +601,7 @@ describe('The example lu files', function () {
     it('Prebuilt models are parsed correctly', function(done) {
         exec(`node ${ludown} parse toluis --in ${TEST_ROOT}/verified/prebuilt_mode.lu --out prebuilt_model_parse.json -o ${TEST_ROOT}/output`, (error, stdout, stderr) => {
             try {
-                assert.deepEqual(txtfile.readSync(TEST_ROOT + '/output/prebuilt_model_parse.json'), txtfile.readSync(TEST_ROOT + '/verified/prebuilt_model_parse.json'));
+                assert.deepEqual(JSON.parse(sanitizeExampleJson(txtfile.readSync(TEST_ROOT + '/output/prebuilt_model_parse.json'))), JSON.parse(sanitizeExampleJson(txtfile.readSync(TEST_ROOT + '/verified/prebuilt_model_parse.json'))));
                 done();
             } catch (err) {
                 done(err);
@@ -623,7 +623,7 @@ describe('The example lu files', function () {
     it('Multiple Prebuilt models are parsed correctly', function(done) {
         exec(`node ${ludown} parse toluis --in ${TEST_ROOT}/verified/calendar_all_prebuilt.lu --out calendar_all_prebuilt_parsed.json -o ${TEST_ROOT}/output`, (error, stdout, stderr) => {
             try {
-                assert.deepEqual(txtfile.readSync(TEST_ROOT + '/output/calendar_all_prebuilt_parsed.json'), txtfile.readSync(TEST_ROOT + '/verified/calendar_all_prebuilt_parsed.json'));
+                assert.deepEqual(JSON.parse(sanitizeExampleJson(txtfile.readSync(TEST_ROOT + '/output/calendar_all_prebuilt_parsed.json'))), JSON.parse(sanitizeExampleJson(txtfile.readSync(TEST_ROOT + '/verified/calendar_all_prebuilt_parsed.json'))));
                 done();
             } catch (err) {
                 done(err);

packages/Ludown/test/ludown.roleTests.test.suite.js

@@ -927,7 +927,7 @@ $test:[fromTime]`;
                 let LUISJSon = res.LUISJsonStructure;
                 assert(LUISJSon.prebuiltEntities.length, 1);
                 assert(LUISJSon.prebuiltEntities[0].roles.length, 2);
-                assert.deepEqual(LUISJSon.prebuiltEntities[0].roles, ['from', 'to']);
+                assert.deepEqual(LUISJSon.prebuiltEntities[0].roles, ['to', 'from']);
                 done();
             })
             .catch (err => done(`Test failed - ${JSON.stringify(err)}`))
@@ -952,7 +952,7 @@ $test:[fromTime]`;
                 let LUISJSon = res.LUISJsonStructure;
                 assert(LUISJSon.regex_entities.length, 1);
                 assert(LUISJSon.regex_entities[0].roles.length, 2);
-                assert.deepEqual(LUISJSon.regex_entities[0].roles, ['from', 'to']);
+                assert.deepEqual(LUISJSon.regex_entities[0].roles, ['to', 'from']);
                 done();
             })
             .catch (err => done(`Test failed - ${JSON.stringify(err)}`))
@@ -979,7 +979,7 @@ $test:[fromTime]`;
                 let LUISJSon = res.LUISJsonStructure;
                 assert(LUISJSon.closedLists.length, 1);
                 assert(LUISJSon.closedLists[0].roles.length, 2);
-                assert.deepEqual(LUISJSon.closedLists[0].roles, ['from', 'to']);
+                assert.deepEqual(LUISJSon.closedLists[0].roles, ['to', 'from']);
                 done();
             })
             .catch (err => done(`Test failed - ${JSON.stringify(err)}`))