Merge pull request #1257 from microsoft/vishwac/1229

[Ludown] Add ability to specify LUIS and QnA application information in .lu file
This commit is contained in:
Emilio Munoz 2019-08-16 08:40:15 -07:00 коммит произвёл GitHub
Родитель d77fc277cc fa15047bea
Коммит 87bb1a520a
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
14 изменённых файлов: 338 добавлений и 33 удалений

Просмотреть файл

@ -115,6 +115,7 @@ After you have bootstrapped and created your LUIS model and / or QnAMaker knowle
--verbose [Optional] Get verbose messages from parser
-s, --skip_header [Optional] Generate .lu file without the header comment
-r, --sort [Optional] When set, intent, utterances, entities, questions collections are alphabetically sorted
-m, --model_info [Optional] When set, include model information in the output .lu file
--prefix [Optional] append [ludown] prefix to all messages
--stdin [Optional] Read input from stdin
--stdout [Optional] Write output to stdout only. Specifying this option will not write any generated content to disk

Просмотреть файл

@ -418,3 +418,20 @@ You can add comments to your .lu document by prefixing the comment with >. Here'
- hi
- hello
```
## Application/ KB information
You can include configuration information for your LUIS application or QnA Maker KB via comments.
**Note** Any information explicitly passed in via CLI arguments will override information in the .lu file.
```markdown
> LUIS application description
> !# @app.name = my luis application
> !# @app.desc = description of my luis application
> !# @app.versionId = 0.5
> !# @app.culture = en-us
> !# @app.luis_schema_version = 3.0.0
> QnA Maker KB description
> !# @kb.name = my qna maker kb name
```

Просмотреть файл

@ -17,6 +17,7 @@ After you have bootstrapped and created your LUIS model and / or QnAMaker knowle
--verbose [Optional] Get verbose messages from parser
-s, --skip_header [Optional] Generate .lu file without the header comment
-r, --sort [Optional] When set, intent, utterances, entities, questions collections are alphabetically sorted
-m, --model_info [Optional] When set, include model information in the output .lu file
--prefix [Optional] append [ludown] prefix to all messages
--stdin [Optional] Read input from stdin
--stdout [Optional] Write output to stdout only. Specifying this option will not write any generated content to disk

Просмотреть файл

@ -13,5 +13,6 @@ module.exports = {
QNATABLE: "|",
ANSWER: "```",
FILTER: "**",
QNAALTERATIONS: "qna-alterations"
QNAALTERATIONS: "qna-alterations",
MODELINFO: "!#"
};

Просмотреть файл

@ -100,7 +100,7 @@ const helpers = {
splitFileBySections : function(fileContent, log) {
fileContent = helpers.sanitizeNewLines(fileContent);
let linesInFile = fileContent.split(NEWLINE);
let currentSection = null;
let currentSection = '';
let middleOfSection = false;
let sectionsInFile = [];
let currentSectionType = null; //PARSERCONSTS
@ -118,7 +118,20 @@ const helpers = {
continue;
}
// skip line if it is just a comment
if(currentLine.indexOf(PARSERCONSTS.COMMENT) === 0) continue;
if(currentLine.indexOf(PARSERCONSTS.COMMENT) === 0) {
// Add support to parse application metadata if found
let info = currentLine.split(/>[ ]*!#/g);
if (info === undefined || info.length === 1) continue;
let previousSection = currentSection.substring(0, currentSection.lastIndexOf(NEWLINE));
try {
sectionsInFile = validateAndPushCurrentBuffer(previousSection, sectionsInFile, currentSectionType, lineIndex, log);
} catch (err) {
throw (err);
}
currentSection = PARSERCONSTS.MODELINFO + info[1].trim() + NEWLINE;
currentSectionType = PARSERCONSTS.MODELINFO;
continue;
}
// skip line if it is blank
if(currentLine === '') continue;
@ -341,6 +354,9 @@ var validateAndPushCurrentBuffer = function(previousSection, sectionsInFile, cur
}
sectionsInFile.push(previousSection);
break;
case PARSERCONSTS.MODELINFO:
sectionsInFile.push(previousSection);
break;
}
return sectionsInFile;
};

Просмотреть файл

@ -23,8 +23,8 @@ program
.option('-s, --subfolder', '[Optional] Include sub-folders as well when looking for .lu files')
.option('-n, --luis_name <luis_appName>', '[Optional] LUIS app name')
.option('-d, --luis_desc <luis_appDesc>', '[Optional] LUIS app description')
.option('-i, --luis_versionId <luis_versionId>', '[Optional] LUIS app version', '0.1')
.option('-c, --luis_culture <luis_appCulture>', '[Optional] LUIS app culture', 'en-us')
.option('-i, --luis_versionId <luis_versionId>', '[Optional] LUIS app version')
.option('-c, --luis_culture <luis_appCulture>', '[Optional] LUIS app culture')
.option('-t, --write_luis_batch_tests', '[Optional] Write out LUIS batch test json file')
.option('--out <OutFileName>', '[Optional] Output file name for the LUIS model')
.option('--verbose', '[Optional] Get verbose messages from parser')

Просмотреть файл

@ -24,6 +24,7 @@ program
.option('--verbose', '[Optional] Get verbose messages from parser')
.option('-s, --skip_header', '[Optional] Generate .lu file without the header comment')
.option('-r, --sort', '[Optional] When set, intent, utterances, entities, questions collections are alphabetically sorted')
.option('-m, --model_info', '[Optional] When set, include model information in the output .lu file')
.option('--stdin', '[Optional] Read input from stdin')
.option('--stdout', '[Optional] Write output to stdout only. Specifying this option will not write any generated content to disk')
.parse(process.argv);

Просмотреть файл

@ -216,6 +216,12 @@ const parseFileContentsModule = {
}
} else if (chunk.indexOf(PARSERCONSTS.QNA) === 0) {
parsedContent.qnaJsonStructure.qnaList.push(new qnaListObj(0, chunkSplitByLine[1], 'custom editorial', [chunkSplitByLine[0].replace(PARSERCONSTS.QNA, '').trim()], []));
} else if (chunk.indexOf(PARSERCONSTS.MODELINFO) === 0) {
try {
parseAndHandleModelInfo(parsedContent, chunkSplitByLine, log);
} catch (err) {
throw (err);
}
}
};
return parsedContent;
@ -268,6 +274,9 @@ const parseFileContentsModule = {
}
});
}
if (blob.name !== undefined) FinalQnAJSON.name = blob.name;
});
return FinalQnAJSON;
},
@ -448,7 +457,23 @@ const parseFileContentsModule = {
}
}
};
const parseAndHandleModelInfo = function(parsedContent, chunkSplitByLine, log) {
// split each line by key value pair
(chunkSplitByLine || []).forEach(line => {
let kvPair = line.split(/@(app|kb).(.*)=/g);
if (kvPair.length === 4) {
if (kvPair[1].trim().toLowerCase() === 'app') {
parsedContent.LUISJsonStructure[kvPair[2].trim()] = kvPair[3].trim();
} else if (kvPair[1].trim().toLowerCase() === 'kb') {
parsedContent.qnaJsonStructure[kvPair[2].trim()] = kvPair[3].trim();
}
} else {
if (log) {
process.stdout.write(chalk.default.yellowBright('[WARN]: Invalid model info found. Skipping "' + line + '"\n'));
}
}
})
};
/**
* Helper function to merge item if it does not already exist
*

Просмотреть файл

@ -78,23 +78,21 @@ const writeOutFiles = function(program,finalLUISJSON,finalQnAJSON, finalQnAAlter
} catch (err) {
throw (err);
}
if(!program.luis_versionId) program.luis_versionId = "0.1";
if(!program.luis_schema_version) program.luis_schema_version = "3.0.0";
if(!program.luis_name) program.luis_name = path.basename(rootFile, path.extname(rootFile));
if(!program.luis_desc) program.luis_desc = "";
if(!program.luis_culture) program.luis_culture = "en-us";
if(!program.qna_name) program.qna_name = path.basename(rootFile, path.extname(rootFile));
if(program.luis_culture) program.luis_culture = program.luis_culture.toLowerCase();
if(finalLUISJSON) {
finalLUISJSON.luis_schema_version = program.luis_schema_version;
finalLUISJSON.versionId = program.luis_versionId;
finalLUISJSON.name = program.luis_name.split('.')[0],
finalLUISJSON.desc = program.luis_desc;
finalLUISJSON.culture = program.luis_culture;
finalLUISJSON.luis_schema_version = program.luis_schema_version || finalLUISJSON.luis_schema_version || "3.0.0";
finalLUISJSON.versionId = program.luis_versionId || finalLUISJSON.versionId || "0.1";
finalLUISJSON.name = program.luis_name || finalLUISJSON.name || path.basename(rootFile, path.extname(rootFile)),
finalLUISJSON.desc = program.luis_desc || finalLUISJSON.desc || "";
finalLUISJSON.culture = program.luis_culture || finalLUISJSON.culture || "en-us";
finalLUISJSON.culture = finalLUISJSON.culture.toLowerCase();
}
if (finalQnAJSON) finalQnAJSON.name = program.qna_name.split('.')[0];
if (!program.luis_name && finalLUISJSON && finalLUISJSON.name) program.luis_name = finalLUISJSON.name;
if (finalQnAJSON) finalQnAJSON.name = program.qna_name || finalQnAJSON.name || path.basename(rootFile, path.extname(rootFile));
if (!program.qna_name && finalQnAJSON && finalQnAJSON.name) program.qna_name = finalQnAJSON.name || '';
var writeQnAFile = (finalQnAJSON.qnaList.length > 0) ||
(finalQnAJSON.urls.length > 0) ||

Просмотреть файл

@ -9,6 +9,22 @@ const helpers = require('./helpers');
const NEWLINE = require('os').EOL;
const exception = require('./classes/exception');
const toLUHelpers = {
constructModelDescFromLUISJSON : async function(LUISJSON) {
let modelDesc = NEWLINE;
modelDesc += '> LUIS application information' + NEWLINE;
modelDesc += '> !# @app.name = ' + LUISJSON.name + NEWLINE;
modelDesc += '> !# @app.desc = ' + LUISJSON.desc + NEWLINE;
modelDesc += '> !# @app.culture = ' + LUISJSON.culture + NEWLINE;
modelDesc += '> !# @app.versionId = ' + LUISJSON.versionId + NEWLINE;
modelDesc += '> !# @app.luis_schema_version = ' + LUISJSON.luis_schema_version + NEWLINE;
return modelDesc;
},
constructModelDescFromQnAJSON : async function(QnAJSON) {
let modelDesc = NEWLINE;
modelDesc += '> QnA KB information' + NEWLINE;
modelDesc += '> !# @kb.name = ' + QnAJSON.name + NEWLINE;
return modelDesc;
},
/**
* Construct lu file content from LUIS JSON object
* @param {object} LUISJSON LUIS JSON object
@ -203,29 +219,40 @@ const toLUHelpers = {
* @param {String} luisFile input LUIS JSON file name
* @param {String} QnAFile input QnA TSV file name
* @param {boolean} skip_header If true, header information in the generated output text will be skipped.
* @param {boolean} include_model_info If true, information about the LUIS/ QnA models are included.
* @returns {String} Generated Markdown file content to flush to disk
* @throws {exception} Throws on errors. exception object includes errCode and text.
*/
constructMdFileHelper : async function(LUISJSON, QnAJSONFromTSV, QnAAltJSON, luisFile, QnAFile, skip_header) {
constructMdFileHelper : async function(LUISJSON, QnAJSONFromTSV, QnAAltJSON, luisFile, QnAFile, skip_header, include_model_info) {
let fileContent = '';
let modelDesc = '';
let fileHeader = '';
let now = new Date();
fileHeader += '> ! Automatically generated by [LUDown CLI](https://github.com/Microsoft/botbuilder-tools/tree/master/Ludown), ' + now.toString() + NEWLINE + NEWLINE;
fileHeader += '> ! Source LUIS JSON file: ' + (LUISJSON.sourceFile?LUISJSON.sourceFile:'Not Specified') + NEWLINE + NEWLINE;
fileHeader += '> ! Source QnA TSV file: ' + (QnAJSONFromTSV.sourceFile?QnAJSONFromTSV.sourceFile:'Not Specified') + NEWLINE + NEWLINE;
fileHeader += '> ! Source QnA Alterations file: ' + (QnAAltJSON.sourceFile?QnAAltJSON.sourceFile:'Not Specified') + NEWLINE + NEWLINE;
if(LUISJSON.sourceFile) {
fileContent += await toLUHelpers.constructMdFromLUISJSON(LUISJSON.model);
modelDesc += await toLUHelpers.constructModelDescFromLUISJSON(LUISJSON.model);
}
if(QnAJSONFromTSV.sourceFile) {
fileContent += await toLUHelpers.constructMdFromQnAJSON(QnAJSONFromTSV.model)
modelDesc += await toLUHelpers.constructModelDescFromQnAJSON(QnAJSONFromTSV.model);
}
if(QnAAltJSON.sourceFile) {
fileContent += await toLUHelpers.constructMdFromQnAAlterationJSON(QnAAltJSON.model)
}
if(fileContent) {
let now = new Date();
if(skip_header) return fileContent
let t = fileContent;
fileContent = '> ! Automatically generated by [LUDown CLI](https://github.com/Microsoft/botbuilder-tools/tree/master/Ludown), ' + now.toString() + NEWLINE + NEWLINE;
fileContent += '> ! Source LUIS JSON file: ' + (LUISJSON.sourceFile?LUISJSON.sourceFile:'Not Specified') + NEWLINE + NEWLINE;
fileContent += '> ! Source QnA TSV file: ' + (QnAJSONFromTSV.sourceFile?QnAJSONFromTSV.sourceFile:'Not Specified') + NEWLINE + NEWLINE;
fileContent += '> ! Source QnA Alterations file: ' + (QnAAltJSON.sourceFile?QnAAltJSON.sourceFile:'Not Specified') + NEWLINE + NEWLINE;
fileContent += t;
if (include_model_info) {
fileContent = modelDesc + fileContent;
}
if(skip_header) {
return fileContent
} else {
return fileHeader + fileContent;
}
}
return fileContent;
},

Просмотреть файл

@ -92,7 +92,7 @@ const toLUModules = {
await toLUHelpers.sortCollections(LUISJSON, QnAJSON, QnAAltJSON);
}
// construct the markdown file content
outFileContent = await toLUHelpers.constructMdFileHelper(LUISJSON, QnAJSON, QnAAltJSON, program.LUIS_File, program.QNA_FILE, program.skip_header)
outFileContent = await toLUHelpers.constructMdFileHelper(LUISJSON, QnAJSON, QnAAltJSON, program.LUIS_File, program.QNA_FILE, program.skip_header, program.model_info)
if(!outFileContent) {
throw(new exception(retCode.errorCode.UNKNOWN_ERROR,'Sorry, Unable to generate .lu file content!'));
}

Просмотреть файл

@ -535,7 +535,7 @@ describe('The example lu files', function () {
it('With -r/ --sort option, ludown refresh correctly sorts a LUIS model', function(done) {
exec(`node ${ludown} refresh -i ${TEST_ROOT}/testcases/all.json -s -r -n luis_sorted.lu -o ${TEST_ROOT}/output`, (error, stdout, stderr) => {
try {
assert.deepEqual(txtfile.readSync(TEST_ROOT + '/output/luis_sorted.lu'), txtfile.readSync(TEST_ROOT + '/verified/luis_sorted.lu'));
compareFiles(TEST_ROOT + '/output/luis_sorted.lu', TEST_ROOT + '/verified/luis_sorted.lu');
done();
} catch (err) {
done(err);
@ -546,7 +546,7 @@ describe('The example lu files', function () {
it('With -r/ --sort option, ludown refresh correctly sorts a QnA model', function(done) {
exec(`node ${ludown} refresh -q ${TEST_ROOT}/testcases/all_qna.json -s -r -n qna_sorted.lu -o ${TEST_ROOT}/output`, (error, stdout, stderr) => {
try {
assert.deepEqual(txtfile.readSync(TEST_ROOT + '/output/qna_sorted.lu'), txtfile.readSync(TEST_ROOT + '/verified/qna_sorted.lu'));
compareFiles(TEST_ROOT + '/output/qna_sorted.lu', TEST_ROOT + '/verified/qna_sorted.lu');
done();
} catch (err) {
done(err);
@ -557,7 +557,7 @@ describe('The example lu files', function () {
it('With -r/ --sort option, ludown refresh correctly sorts a QnA Alteration model', function(done) {
exec(`node ${ludown} refresh -a ${TEST_ROOT}/testcases/qna-alterations_Alterations.json -s -r -n qna_a_sorted.lu -o ${TEST_ROOT}/output`, (error, stdout, stderr) => {
try {
assert.deepEqual(txtfile.readSync(TEST_ROOT + '/output/qna_a_sorted.lu'), txtfile.readSync(TEST_ROOT + '/verified/qna_a_sorted.lu'));
compareFiles(TEST_ROOT + '/output/qna_a_sorted.lu', TEST_ROOT + '/verified/qna_a_sorted.lu');
done();
} catch (err) {
done(err);
@ -568,11 +568,23 @@ describe('The example lu files', function () {
it('With -r/ --sort option, ludown refresh correctly sorts with LUIS, QnA and QnA alternation models are specified as input', function(done){
exec(`node ${ludown} refresh -i ${TEST_ROOT}/testcases/all.json -q ${TEST_ROOT}/testcases/all_qna.json -a ${TEST_ROOT}/testcases/qna-alterations_Alterations.json -s -r -n sorted.lu -o ${TEST_ROOT}/output`, (error, stdout, stderr) => {
try {
assert.deepEqual(txtfile.readSync(TEST_ROOT + '/output/sorted.lu'), txtfile.readSync(TEST_ROOT + '/verified/sorted.lu'));
compareFiles(TEST_ROOT + '/output/sorted.lu', TEST_ROOT + '/verified/sorted.lu');
done();
} catch (err) {
done(err);
}
});
});
it('With -m/ --model_info option, ludown refresh correctly writes out model information in output', function(done) {
exec(`node ${ludown} refresh -i ${TEST_ROOT}/testcases/all.json -m -s -r -n modelInfo.lu -o ${TEST_ROOT}/output`, (error, stdout, stderr) => {
try {
compareFiles(TEST_ROOT + '/output/modelInfo.lu', TEST_ROOT + '/verified/modelInfo.lu');
done();
} catch (err) {
done(err);
}
});
})
});

Просмотреть файл

@ -896,6 +896,77 @@ describe('parseFile correctly parses utterances', function () {
})
.catch(err => done(err))
});
it ('application meta data information in lu file is parsed correctly', function(done) {
let testLU = `> !# @app.name = all345
> !# @app.desc = this is a test
> !# @app.culture = en-us
> !# @app.versionId = 0.4
> !# @app.luis_schema_version = 3.0.0
# test
- greeting`;
parseFile.parseFile(testLU)
.then(res => {
assert.equal(res.LUISJsonStructure.name, 'all345');
assert.equal(res.LUISJsonStructure.desc, 'this is a test');
assert.equal(res.LUISJsonStructure.culture, 'en-us');
assert.equal(res.LUISJsonStructure.versionId, '0.4');
assert.equal(res.LUISJsonStructure.luis_schema_version, '3.0.0')
done();
})
.catch(err => done(err))
});
it ('kb meta data information in lu file is parsed correctly', function(done) {
let testLU = `> !# @kb.name = my test kb
# ? hi
\`\`\`markdown
hello
\`\`\``;
parseFile.parseFile(testLU)
.then(res => {
assert.equal(res.qnaJsonStructure.name, 'my test kb');
done();
})
.catch(err => done(err))
})
it ('LUIS and QnA meta data information in lu file is parsed correctly', function(done){
let testLU = `> !# @kb.name = my test kb
# ? hi
\`\`\`markdown
hello
\`\`\`
> !# @app.versionId = 0.6
> !# @app.name = orange tomato
# test
- greeting`;
parseFile.parseFile(testLU)
.then(res => {
assert.equal(res.qnaJsonStructure.name, 'my test kb');
assert.equal(res.LUISJsonStructure.name, 'orange tomato');
assert.equal(res.LUISJsonStructure.versionId, '0.6');
done();
})
.catch(err => done(err))
})
it ('Multi line app meta data definition throws correctly', function(done){
let testLU = `> !# @kb.name = foo bar
test
# ? test q`;
parseFile.parseFile(testLU)
.then(res => done(`Did not throw when expected`))
.catch(err => done())
})
})

Просмотреть файл

@ -0,0 +1,135 @@
> LUIS application information
> !# @app.name = all
> !# @app.desc =
> !# @app.culture = en-us
> !# @app.versionId = 0.1
> !# @app.luis_schema_version = 3.0.0
> # Intent definitions
## AskForUserName
- {userName=vishwac}
- I'm {userName=vishwac}
- call me {userName=vishwac}
- my name is {userName=vishwac}
- {userName=vishwac} is my name
- you can call me {userName=vishwac}
## Buy chocolate
- I would like to buy some kit kat
- I want some twix
- can I get some m&m
## CommunicationPreference
- set phone call as my communication preference
- I prefer to receive text message
## CreateAlarm
- create an alarm for 7AM
- create an alarm
- set an alarm for 7AM next thursday
## DeleteAlarm
- delete the {alarmTime} alarm
- remove the {alarmTime} alarm
## Greeting
- Hi
- Good morning
- Good evening
- Hello
## Help
- can you help
- please help
- I need help
- help
## None
- who is your ceo?
- santa wants a blue ribbon
## setThermostat
- Please set {deviceTemperature=thermostat to 72}
- Set {deviceTemperature={customDevice=owen} to 72}
## testIntent
- I need a flight from {datetimeV2:fromDate=tomorrow} and returning on {datetimeV2:toDate=next thursday}
> # Entity definitions
$customDevice:simple
$userName:simple
> # PREBUILT Entity definitions
$PREBUILT:age
$PREBUILT:datetimeV2 Roles=fromDate, toDate
$PREBUILT:temperature
> # Phrase list definitions
$ChocolateType:phraseList
- m&m,mars,mints,spearmings,payday,jelly,kit kat,kitkat,twix
$question:phraseList interchangeable
- are you,you are
> # List entities
$commPreference:call=
- phone call
- give me a ring
- ring
- call
- cell phone
- phone
$commPreference:text=
- message
- text
- sms
- text message
$commPreference:fax=
- fax
- fascimile
$device:thermostat=
- Thermostat
- Heater
- AC
- Air conditioner
$device:refrigerator=
- Fridge
- Cooler
> # RegEx entities
$HRF-number:/hrf-[0-9]{6}/
$zander:/z-[0-9]{3}/
> # Composite entities
$deviceTemperature:[device, customDevice, temperature]
$units:[temperature]