microsoft
/
botbuilder-tools
зеркало из https://github.com/microsoft/botbuilder-tools.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

[Ludown] fix validate luis json api to not modify source json structure (#1254) 

* Fix for 1230

* Fix for #1232

* Fixing tests that fail on windows

* Fix broken tests

* Updating tests
This commit is contained in:
vishwacsena 2019-08-19 17:09:06 -07:00 коммит произвёл GitHub
Родитель fb6a5e4a79
Коммит 643feb260c
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
4 изменённых файлов: 110 добавлений и 32 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

87
packages/Ludown/lib/parseFileContents.js
Просмотреть файл

@ -31,17 +31,7 @@ const parseFileContentsModule = {
* @throws {exception} Throws on errors. exception object includes errCode and text.
*/
validateLUISBlob: async function (LUISJSONBlob) {
// patterns can have references to any other entity types.
// So if there is a pattern.any entity that is also defined as another type, remove the pattern.any entity
LUISJSONBlob.patternAnyEntities = (LUISJSONBlob.patternAnyEntities || []).filter(entity => {
if (itemExists(LUISJSONBlob.entities, entity.name, entity.roles)) return false;
if (itemExists(LUISJSONBlob.closedLists, entity.name, entity.roles)) return false;
if (itemExists(LUISJSONBlob.model_features, entity.name, entity.roles)) return false;
if (itemExists(LUISJSONBlob.prebuiltEntities, entity.name, entity.roles)) return false;
if (itemExists(LUISJSONBlob.regex_entities, entity.name, entity.roles)) return false;
if (itemExists(LUISJSONBlob.prebuiltEntities, entity.name, entity.roles)) return false;
return true;
});
// look for entity name collisions - list, simple, patternAny, phraselist
// look for list entities labelled
@ -380,6 +370,40 @@ const parseFileContentsModule = {
}
}
});
// do we have pattern.any entities here?
(blob.patternAnyEntities || []).forEach(patternAny => {
let paIdx = -1;
let patternAnyInMaster = FinalLUISJSON.patternAnyEntities.find((item, idx) => {
if (item.name === patternAny.name) {
paIdx = idx;
return true;
}
return false;
});
// verify that this patternAny entity does not exist as any other type
let simpleEntityInMaster = FinalLUISJSON.entities.find(item => item.name == patternAny.name);
let compositeInMaster = FinalLUISJSON.composites.find(item => item.name == patternAny.name);
let listEntityInMaster = FinalLUISJSON.closedLists.find(item => item.name == patternAny.name);
let regexEntityInMaster = FinalLUISJSON.regex_entities.find(item => item.name == patternAny.name);
let prebuiltInMaster = FinalLUISJSON.prebuiltEntities.find(item => item.name == patternAny.name);
if (!simpleEntityInMaster &&
!compositeInMaster &&
!listEntityInMaster &&
!regexEntityInMaster &&
!prebuiltInMaster) {
if (patternAnyInMaster) {
(patternAny.roles || []).forEach(role => !patternAnyInMaster.roles.includes(role) ? patternAnyInMaster.roles.push(role) : undefined);
} else {
FinalLUISJSON.patternAnyEntities.push(patternAny);
}
} else {
// remove the pattern.any from master if another entity type has this name.
if (patternAnyInMaster) {
if (paIdx !== -1) FinalLUISJSON.patternAnyEntities.splice(paIdx, 1);
}
}
})
});
return FinalLUISJSON;
},
@ -645,16 +669,22 @@ const parseAndHandleEntity = function (parsedContent, chunkSplitByLine, locale,
let pEntityName = (entityName === 'PREBUILT') ? entityType : entityName;
// see if we already have this as Pattern.Any entity
// see if we already have this in patternAny entity collection; if so, remove it but remember the roles (if any)
for (let i in parsedContent.LUISJsonStructure.patternAnyEntities) {
if (parsedContent.LUISJsonStructure.patternAnyEntities[i].name === pEntityName) {
let patternAnyMatchIdx = new Array();
(parsedContent.LUISJsonStructure.patternAnyEntities || []).forEach((entity, idx) => {
if (entity.name === pEntityName || entity.name === entityType) {
if (entityType.toLowerCase().trim().includes('phraselist')) {
throw (new exception(retCode.errorCode.INVALID_INPUT, '[ERROR]: Phrase lists cannot be used as an entity in a pattern "' + pEntityName));
}
if (parsedContent.LUISJsonStructure.patternAnyEntities[i].roles.length !== 0) entityRoles = parsedContent.LUISJsonStructure.patternAnyEntities[i].roles;
parsedContent.LUISJsonStructure.patternAnyEntities.splice(i, 1);
break;
if (entity.roles.length !== 0) entityRoles = entity.roles;
patternAnyMatchIdx.push(idx);
}
}
});
// remove any found patternany entities.
patternAnyMatchIdx.sort().reverse().forEach(idx => {
parsedContent.LUISJsonStructure.patternAnyEntities.splice(idx, 1);
})
// add this entity to appropriate place
// is this a builtin type?
if (builtInTypes.consolidatedList.includes(entityType)) {
@ -1076,13 +1106,24 @@ const parseAndHandleIntent = function (parsedContent, chunkSplitByLine) {
if (!parsedContent.LUISJsonStructure.patterns.find(item => deepEqual(item, newPattern))) {
parsedContent.LUISJsonStructure.patterns.push(newPattern);
}
// add all entities to pattern.Any.
// add all entities to pattern.Any only if they do not have another type.
entitiesFound.forEach(entity => {
if (entity.role !== '') {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity.entity, [entity.role.trim()])
} else {
addItemIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity.entity);
}
let simpleEntityInMaster = parsedContent.LUISJsonStructure.entities.find(item => item.name == entity.entity);
let compositeInMaster = parsedContent.LUISJsonStructure.composites.find(item => item.name == entity.entity);
let listEntityInMaster = parsedContent.LUISJsonStructure.closedLists.find(item => item.name == entity.entity);
let regexEntityInMaster = parsedContent.LUISJsonStructure.regex_entities.find(item => item.name == entity.entity);
let prebuiltInMaster = parsedContent.LUISJsonStructure.prebuiltEntities.find(item => item.name == entity.entity);
if (!simpleEntityInMaster &&
!compositeInMaster &&
!listEntityInMaster &&
!regexEntityInMaster &&
!prebuiltInMaster) {
if (entity.role !== '') {
addItemOrRoleIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity.entity, [entity.role.trim()])
} else {
addItemIfNotPresent(parsedContent.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity.entity);
}
}
});
// if this intent does not have any utterances, push this pattern as an utterance as well.

45
packages/Ludown/lib/parser.js
Просмотреть файл

@ -418,13 +418,50 @@ const resolveReferencesInUtterances = async function(allParsedContent) {
entitiesFound.forEach(function (entity) {
entity = entity.replace("{", "").replace("}", "");
let entityName = entity;
let roleName = '';
if (entity.includes(':')) {
// this is an entity with role
const [entityName, roleName] = entity.split(':');
parseFileContents.addItemOrRoleIfNotPresent(luisModel.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entityName, [roleName])
[entityName, roleName] = entity.split(':');
}
// insert the entity only if it does not already exist
let simpleEntityInMaster = luisModel.LUISJsonStructure.entities.find(item => item.name == entityName);
let compositeInMaster = luisModel.LUISJsonStructure.composites.find(item => item.name == entityName);
let listEntityInMaster = luisModel.LUISJsonStructure.closedLists.find(item => item.name == entityName);
let regexEntityInMaster = luisModel.LUISJsonStructure.regex_entities.find(item => item.name == entityName);
let prebuiltInMaster = luisModel.LUISJsonStructure.prebuiltEntities.find(item => item.name == entityName);
let paIdx = -1;
let patternAnyInMaster = luisModel.LUISJsonStructure.patternAnyEntities.find((item, idx) => {
if (item.name === entityName) {
paIdx = idx;
return true;
}
return false;
});
if (!simpleEntityInMaster &&
!compositeInMaster &&
!listEntityInMaster &&
!regexEntityInMaster &&
!prebuiltInMaster) {
if (!patternAnyInMaster) {
// add a pattern.any entity
if (roleName !== '') {
parseFileContents.addItemOrRoleIfNotPresent(luisModel.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entityName, [roleName])
} else {
parseFileContents.addItemIfNotPresent(luisModel.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity);
}
} else {
// add the role if it does not exist already.
if (roleName !== '') {
!patternAnyInMaster.roles.includes(roleName) ? patternAnyInMaster.roles.push(roleName) : undefined;
}
}
} else {
parseFileContents.addItemIfNotPresent(luisModel.LUISJsonStructure, LUISObjNameEnum.PATTERNANYENTITY, entity);
// we found this pattern.any entity as another type.
if (patternAnyInMaster && paIdx !== -1) {
// remove the patternAny entity from the list because it has been explicitly defined elsewhere.
luisModel.LUISJsonStructure.patternAnyEntities.splice(paIdx, 1);
}
}
});
}

4
packages/Ludown/test/ludown.examples.test.suite.js
Просмотреть файл

@ -601,7 +601,7 @@ describe('The example lu files', function () {
it('Prebuilt models are parsed correctly', function(done) {
exec(`node ${ludown} parse toluis --in ${TEST_ROOT}/verified/prebuilt_mode.lu --out prebuilt_model_parse.json -o ${TEST_ROOT}/output`, (error, stdout, stderr) => {
try {
assert.deepEqual(txtfile.readSync(TEST_ROOT + '/output/prebuilt_model_parse.json'), txtfile.readSync(TEST_ROOT + '/verified/prebuilt_model_parse.json'));
assert.deepEqual(JSON.parse(sanitizeExampleJson(txtfile.readSync(TEST_ROOT + '/output/prebuilt_model_parse.json'))), JSON.parse(sanitizeExampleJson(txtfile.readSync(TEST_ROOT + '/verified/prebuilt_model_parse.json'))));
done();
} catch (err) {
done(err);
@ -623,7 +623,7 @@ describe('The example lu files', function () {
it('Multiple Prebuilt models are parsed correctly', function(done) {
exec(`node ${ludown} parse toluis --in ${TEST_ROOT}/verified/calendar_all_prebuilt.lu --out calendar_all_prebuilt_parsed.json -o ${TEST_ROOT}/output`, (error, stdout, stderr) => {
try {
assert.deepEqual(txtfile.readSync(TEST_ROOT + '/output/calendar_all_prebuilt_parsed.json'), txtfile.readSync(TEST_ROOT + '/verified/calendar_all_prebuilt_parsed.json'));
assert.deepEqual(JSON.parse(sanitizeExampleJson(txtfile.readSync(TEST_ROOT + '/output/calendar_all_prebuilt_parsed.json'))), JSON.parse(sanitizeExampleJson(txtfile.readSync(TEST_ROOT + '/verified/calendar_all_prebuilt_parsed.json'))));
done();
} catch (err) {
done(err);

6
packages/Ludown/test/ludown.roleTests.test.suite.js
Просмотреть файл

@ -927,7 +927,7 @@ $test:[fromTime]`;
let LUISJSon = res.LUISJsonStructure;
assert(LUISJSon.prebuiltEntities.length, 1);
assert(LUISJSon.prebuiltEntities[0].roles.length, 2);
assert.deepEqual(LUISJSon.prebuiltEntities[0].roles, ['from', 'to']);
assert.deepEqual(LUISJSon.prebuiltEntities[0].roles, ['to', 'from']);
done();
})
.catch (err => done(`Test failed - ${JSON.stringify(err)}`))
@ -952,7 +952,7 @@ $test:[fromTime]`;
let LUISJSon = res.LUISJsonStructure;
assert(LUISJSon.regex_entities.length, 1);
assert(LUISJSon.regex_entities[0].roles.length, 2);
assert.deepEqual(LUISJSon.regex_entities[0].roles, ['from', 'to']);
assert.deepEqual(LUISJSon.regex_entities[0].roles, ['to', 'from']);
done();
})
.catch (err => done(`Test failed - ${JSON.stringify(err)}`))
@ -979,7 +979,7 @@ $test:[fromTime]`;
let LUISJSon = res.LUISJsonStructure;
assert(LUISJSon.closedLists.length, 1);
assert(LUISJSon.closedLists[0].roles.length, 2);
assert.deepEqual(LUISJSon.closedLists[0].roles, ['from', 'to']);
assert.deepEqual(LUISJSon.closedLists[0].roles, ['to', 'from']);
done();
})
.catch (err => done(`Test failed - ${JSON.stringify(err)}`))