Merge pull request #6 from v-geberr/master

Using public app id

.vscode/launch.json

@@ -0,0 +1,14 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "node",
+            "request": "launch",
+            "name": "Launch Program",
+            "program": "${file}"
+        }
+    ]
+}

documentation-samples/endpoint-api-samples/python/quickstart-call-endpoint-3-6.py

@@ -1,8 +1,6 @@
 ########### Python 3.6 #############
 import requests
 
-appid = '<app id>'
-
 headers = {
     # Request headers
     'Ocp-Apim-Subscription-Key': 'YOUR-SUBSCRIPTION-KEY',
@@ -19,7 +17,7 @@ params ={
 }
 
 try:
-    r = requests.get('https://westus.api.cognitive.microsoft.com/luis/v2.0/apps/' + appid,headers=headers, params=params)
+    r = requests.get('https://westus.api.cognitive.microsoft.com/luis/v2.0/apps/60340f5f-99c1-4043-8ab9-c810ff16252d',headers=headers, params=params)
     print(r.json())
 
 except Exception as e:

examples/README.md

@@ -10,6 +10,10 @@ These samples provide LUIS apps to demonstrate scenarios. You can import the LUI
   [PhraseListDoc]: https://docs.microsoft.com/en-us/azure/cognitive-services/LUIS/add-features#phrase-list-features
   [LUIS]: https://www.luis.ai
 
-* [Batch Upload Utterances Example][BatchUpload]: The batch upload utterances example demonstrates how to download the LUIS query log, parse the log into a batch of utterances, then upload the batch back to LUIS for testing and training purposes.  
+* [Batch Upload Utterances Example][BatchUpload]: The batch upload utterances example demonstrates how to download the LUIS query log, parse the log into a batch of utterances, then upload the batch back to LUIS for testing and training purposes. You must provide a LUIS subscription key and LUIS application id.
 
   [BatchUpload]: ./demo-Upload-utterances-from-querylog/readme.md
+
+* [Batch Upload IOT Lights Example][BatchIOTUpload]: The batch upload utterances example demonstrates how to use queries from other tools, parse the log into a batch of utterances, then upload the batch back to LUIS for testing and training purposes. You must provide a LUIS subscription key and LUIS application id.
+
+  [BatchIOTUpload]: ./demo-Upload-utterances-from-iot-lights/readme.md

examples/demo-Upload-utterances-from-iot-lights/_parse.js

@@ -0,0 +1,95 @@
+// node 7.x
+// built with streams for larger files
+
+const fse = require('fs-extra');
+const path = require('path');
+const lineReader = require('line-reader');
+const Promise = require('bluebird');
+
+
+// rewrite each items properties and values
+function mapEntity(entities) {
+
+    return entities.map(entity => {
+
+        // create new properties
+        entity.entityName = entity.entity;
+        entity.startCharIndex = entity.startPos;
+        entity.endCharIndex = entity.endPos;
+
+        // delete old properties
+        delete entity.startPos;
+        delete entity.endPos;
+        delete entity.entity;
+
+        return entity;
+
+    });
+}
+
+var utterance = function (i, item) {
+
+    let json = {
+        "row": i,
+        "text": "",
+        "intentName": "",
+        "entityLabels": {}
+    };
+
+    if (!item) return json;
+
+    try {
+
+        json.intentName = item.intent;
+        json.text = item.text;
+        json.entityLabels = item.entities && item.entities.length ? mapEntity(item.entities) : [];
+
+        return json;
+
+    } catch (err) {
+        // do something with error
+        console.log("err " + err);
+    }
+
+};
+
+// main function to call
+// read stream line at a time
+// conversion happens in precess.convert_utterance_map file
+const convert = async (config) => {
+
+    try{
+
+        var i = 0;
+
+        // get inFile json
+        inFileJSON = await fse.readFile(config.inFile, 'utf-8');
+
+        // create out file
+        var myOutFile = await fse.createWriteStream(config.outFile, 'utf-8');
+        myOutFile.write('[');
+
+        // read 1 utterance
+        inFileJSON.utterances.forEach( (item) => {
+
+            // transform utterance from original json to LUIS batch json
+            jsonUtterance = utterance(++i, item);
+
+            // write to out stream
+            if (i > 1) myOutFile.write(",");
+            myOutFile.write(JSON.stringify(jsonUtterance));
+
+        });
+        
+        myOutFile.write(']');
+        myOutFile.end();
+        console.log("parse done");
+        return config;
+
+    }catch (err) {
+        return err;
+    }
+
+}
+
+module.exports = convert;

examples/demo-Upload-utterances-from-iot-lights/_upload.js

@@ -0,0 +1,77 @@
+// node 7.x
+// uses async/await - promises
+
+var rp = require('request-promise');
+var fs = require('fs-extra');
+var path = require('path');
+
+
+// main function to call
+var upload = async (config) => {
+
+    try{
+    // request options
+    config.options = {
+        uri: config.uri,
+        method: 'POST',
+        headers: {
+            'Ocp-Apim-Subscription-Key': config.LUIS_subscriptionKey
+        },
+        json: true
+    };
+
+    config.response = {
+        success: {},
+        error: {}
+    };
+
+    return await getBatchFromFile(config)
+        .then(sendBatchToApi)
+        .then(response => {
+            console.log("upload done");
+            return response;
+        });
+        
+    } catch(err){
+        return err;        
+    }
+
+}
+// get json from file - already formatted for this api
+var getBatchFromFile = async (config) => {
+    try {
+
+        var inFile = await fs.readFile(config.inFile, 'utf-8');
+
+        config.options.body = JSON.parse(inFile);
+        config.response.success.getBatchFromFile = true;
+
+        return config;
+
+    } catch (err) {
+        console.log(err);
+        config.response.error.getBatchFromFile = err;
+        return config;
+    }
+}
+// send json as post.body to api
+var sendBatchToApi = async (config) => {
+    try {
+
+        uploadResponse = await rp.post(config.options);       
+        writeFileResponse = await fs.writeFile(config.inFile.replace('.json','.upload.json'),JSON.stringify(uploadResponse), 'utf-8');
+        
+        config.response.success.sendBatchToApi = {};
+        config.response.success.sendBatchToApi.upload = uploadResponse;
+        config.response.success.sendBatchToApi.writeFile = writeFileResponse;
+        
+        return config;
+    }
+    catch (err) {
+        config.response.error.sendBatchToApi = err;
+        console.log(JSON.stringify(err));
+        return err;
+    }
+}
+
+module.exports = upload;

examples/demo-Upload-utterances-from-iot-lights/example-files/iot-lights-utterances.json

@@ -0,0 +1,25 @@
+{
+    "utterances": [
+        {
+            "text": "turn the lights off",
+            "intent": "TurnAllOff",
+            "entities": []
+        },
+        {
+            "text": "turn on floor lamp in green",
+            "intent": "TurnOn",
+            "entities": [
+                {
+                    "entity": "Light",
+                    "startPos": 8,
+                    "endPos": 12
+                },
+                {
+                    "entity": "Color",
+                    "startPos": 22,
+                    "endPos": 26
+                }
+            ]
+        }
+    ]
+}

examples/demo-Upload-utterances-from-iot-lights/index.js

@@ -0,0 +1,44 @@
+var path = require('path');
+
+const parse = require('./_parse');
+const upload = require('./_upload');
+
+// TBD: CHANGE THESE VALUES
+const LUIS_subscriptionKey = "YOUR_SUBSCRIPTION_KEY";
+const LUIS_appId = "YOUR_APP_ID";
+
+const LUIS_versionId = "0.1";
+
+// NOTE: final output of upload api named utterances.upload.json
+const iotFile= "./iot-lights-utterances.json";
+const uploadFile = "./utterances.json"
+
+/* upload configuration */
+var configUpload = {
+    LUIS_subscriptionKey: LUIS_subscriptionKey,
+    LUIS_appId: LUIS_appId,
+    LUIS_versionId: LUIS_versionId,
+    inFile: path.join(__dirname, uploadFile),
+    uri: "https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/{appId}/versions/{versionId}/examples".replace("{appId}", LUIS_appId).replace("{versionId}", LUIS_versionId)
+};
+
+/* parse configuration */
+var configParse = {
+    inFile: path.join(__dirname, iotFile),
+    outFile: path.join(__dirname, uploadFile)
+}; 
+
+var output = {};
+
+parse(configParse)
+.then(output => {
+    output.convert = output;
+    return upload(configUpload);
+}).then(output => {
+    output.upload = output;
+    console.log("process done");  
+});
+
+// single step - uncomment 1 line only
+//parse(configParse);
+//upload(configUpload)

examples/demo-Upload-utterances-from-iot-lights/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "demo-Upload-utterances-from-iot-lights",
+  "version": "1.0.0",
+  "description": "Use Microsoft LUIS api to upload utterance examples from IOT Lights.",
+  "main": "index.js",
+  "scripts": {
+    "start": "node index.js"
+  },
+  "author": "v-geberr",
+  "license": "ISC",
+  "dependencies": {
+    "bluebird": "^3.5.1",
+    "fs-extra": "^4.0.2",
+    "line-reader": "^0.4.0",
+    "request": "^2.83.0",
+    "request-promise": "^4.2.2"
+  }
+}

examples/demo-Upload-utterances-from-iot-lights/readme.md

@@ -0,0 +1,186 @@
+# Upload utterances from IOT Lights
+
+A sample nodeJs application to use queries from a hypothetical IOT application. This application parses the IOT queries into the acceptable LUIS query format, and [uploads as a batch](https://westus.dev.cognitive.microsoft.com/docs/services/5890b47c39e2bb17b84a55ff/operations/5890b47c39e2bb052c5b9c09).
+
+The application main file is the [index.js]('./index.js). This file contains the configuration settings and calls into the three files:
+
+- [_parse.js](./_parse.js) : convert csv from query logs into json for upload
+- [_upload.js](./_upload) : upload json to batch label api
+
+The application assumes the IOT queries are in a JSON file format. An example is found at:
+
+- [./example-files/iot-lights-utterances.json](./example-files/iot-lights-utterances.json) : convert json from IOT logs into json for upload
+
+The application will create files associated with each step:
+
+- [utterances.json](./example-files/utterances.json) : batch labels
+- [utterances.upload.json](./example-files/utterances.upload.json) : final response body from upload api
+
+If one or all of these files is missing, their was an error with the application. 
+
+### Prerequisites
+The minimum prerequisites to run this sample are:
+* Latest Node.js with NPM. Download it from [here](https://nodejs.org/en/download/).
+* A [trained](https://docs.microsoft.com/en-us/azure/cognitive-services/LUIS/train-test) and [published](https://docs.microsoft.com/en-us/azure/cognitive-services/LUIS/publishapp) LUIS Application. 
+* **[Recommended]** Visual Studio Code for IntelliSense and debugging, download it from [here](https://code.visualstudio.com/) for free.
+
+### Install
+Install the Node.js dependencies from NPM in the terminal/command line.
+
+````
+> npm install
+````
+
+### Change Configuration Settings
+In order to use this application, you need to change the values in the index.js file to your own subscription key, app ID, and version ID.
+
+Open the index.js file, and change these values at the top of the file.
+
+````JavaScript
+// TBD: CHANGE THESE VALUES
+const LUIS_subscriptionKey = "YOUR_SUBSCRIPTION_KEY";
+const LUIS_appId = "YOUR_APP_ID";
+const LUIS_versionId = "0.1";
+````
+### Run the application
+Run the application from a terminal/command line with Node.js.
+
+````
+> node index.js
+````
+### Application progress
+While the application is running, the terminal/command line will show progress.
+
+````
+> node index.js
+download done
+parse done
+upload done
+process done
+````
+
+### LUIS Apis used in this sample
+This sample uses the [batch add labels](https://westus.dev.cognitive.microsoft.com/docs/services/5890b47c39e2bb17b84a55ff/operations/5890b47c39e2bb052c5b9c09) api.
+
+### Format of the JSON for the batch upload
+The format of the JSON for the batch upload is noted in the [batch add labels](https://westus.dev.cognitive.microsoft.com/docs/services/5890b47c39e2bb17b84a55ff/operations/5890b47c39e2bb052c5b9c09) api.
+
+### Use your own private apps
+If you incorrectly use an app ID that you do not have permission to upload to, such as any public apps, you will recieve an error.
+
+### Intent And Entities are not created if NOT found
+Any intent or entity uploaded that is not found in your LUIS app will cause an error. It is important that all intents and entities used in the batch already exist in the app.
+
+### Errors in output file of the application
+The final response body from upload api is in the 'utterances.upload.json' file. This file will be an array of responses, one response for each item in the batch.
+
+Each item in the batch can succeed or fail independent of any other item, so it is important to check the response.
+
+#### Examples of correctly formatted items:
+
+````JavaScript
+// successfully formated item
+    {
+        "row": 1,
+        "text": "go to paris",
+        "intentName": "BookFlight",
+        "entityLabels": [
+            {
+                "entityName": "Location::LocationTo",
+                "startCharIndex": 6,
+                "endCharIndex": 10
+            }
+        ]
+    }
+````
+
+#### An example of a successful item upload response:
+
+````JavaScript
+// successfully uploaded item
+{
+    "value": {
+        "UtteranceText": "go to paris",
+        "ExampleId": -175128
+    },
+    "hasError": false
+}
+````
+
+#### Examples of successful request with failed items :
+
+````JavaScript
+// failed uploaded item - don't upload built-ins
+{
+    "value": null,
+    "hasError": true,
+    "error": {
+        "code": "FAILED",
+        "message": "ticket to seattle tomorrowtimezoneOffset=0. Error: The entity extractor builtin.number doesn't exist in the selected application"
+    }
+}
+````
+
+````JavaScript
+// failed uploaded item - missing intent
+{
+    "value": null,
+    "hasError": true,
+    "error": {
+        "code":"FAILED","message":"turn on the left light. Error: The intent classifier TurnOn does not exist in the selected application"
+    }
+}
+````
+ 
+#### Examples of failed requests because of malformed items
+
+Batch upload items (or the whole batch) can result in parsing errors in the LUIS api. These errors are generally returned as HTTP 400 status errors instead of returning a successful response with an array of items, some of which failed.
+
+````JavaScript
+// malformed item - entityLabels first array item is present but empty
+// fix - should remove {}
+{
+    "row": 2,
+    "text": "ticket to paris",
+    "intentName": "BookFlight",
+    "entityLabels": [
+        {
+            
+        }
+    ]
+}
+
+
+// Http Error 400 - 
+
+```` 
+
+````JavaScript
+// malformed item - malformed JSON - no comma
+// fix - add comma after every key:value pair
+[
+    {
+        "text": "Hello"
+        "intent": "Greetings"
+    },
+    {
+        "text": "I want bread"
+        "intent": "Request"
+    }
+]
+```` 
+
+````JavaScript
+// malformed item - malformed JSON - extra comman at end of key:value pair
+// fix - remove extra comma
+[
+    {
+        "text": "Hello",
+        "intent": "Greetings",
+    },
+    {
+        "text": "I want bread",
+        "intent": "Request"
+    }
+]
+````

examples/demo-Upload-utterances-from-querylog/_parse.js

@@ -99,8 +99,6 @@ const convert = async (config) => {
             // skip first line with headers
             if (i++ == 0) return;
 
-            //console.log(i);
-
             // transform utterance from csv to json
             jsonUtterance = utterance((i - 1), line);
 

examples/demo-Upload-utterances-from-querylog/_upload.js

@@ -24,7 +24,7 @@ var upload = async (config) => {
         success: {},
         error: {}
     };
-    //console.log("upload");
+
     return await getBatchFromFile(config)
         .then(sendBatchToApi)
         .then(response => {
@@ -40,14 +40,9 @@ var upload = async (config) => {
 // get json from file - already formatted for this api
 var getBatchFromFile = async (config) => {
     try {
-        //console.log("getBatchFromFile");
-        //console.log(config.inFile);
 
         var inFile = await fs.readFile(config.inFile, 'utf-8');
-
-        //console.log(inFile);
         config.options.body = JSON.parse(inFile);
-        //console.log(config.options.body);
         config.response.success.getBatchFromFile = true;
 
         return config;
@@ -74,7 +69,6 @@ var sendBatchToApi = async (config) => {
     catch (err) {
         config.response.error.sendBatchToApi = err;
         console.log(JSON.stringify(err));
-        //console.log("sendBatchToApi failed = " + err.response.statusCode + " " + err.response.statusMessage);
         return err;
     }
 }

examples/demo-Upload-utterances-from-querylog/index.js

@@ -5,8 +5,8 @@ const parse = require('./_parse');
 const upload = require('./_upload');
 
 // TBD: CHANGE THESE VALUES
-const LUIS_subscriptionKey = "e237d6bc86cd4562bf67b09dff44d2e6"; 
+const LUIS_subscriptionKey = "YOUR_SUBSCRIPTION_KEY"; 
-const LUIS_appId = "84d6601f-a1f0-456e-a894-be5e662a5a6b";
+const LUIS_appId = "YOUR_APP_ID";
 const LUIS_versionId = "0.1";
 
 // NOTE: final output of upload api named utterances.upload.json
@@ -17,7 +17,7 @@ const uploadFile = "./utterances.json"
 var configDownload = {
     LUIS_subscriptionKey: LUIS_subscriptionKey,
     LUIS_appId: LUIS_appId,
-    outFile: path.join(__dirname, "./utterances.csv"),
+    outFile: path.join(__dirname, downloadFile),
     uri: "https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/{appId}/querylogs".replace("{appId}",LUIS_appId)
 };
 
@@ -32,8 +32,8 @@ var configUpload = {
 
 /* parse configuration */
 var configParse = {
-    inFile: path.join(__dirname, "./utterances.csv"),
+    inFile: path.join(__dirname, downloadFile),
-    outFile: path.join(__dirname, "./utterances.json")
+    outFile: path.join(__dirname, uploadFile)
 }; 
 
 var output = {};

examples/demo-Upload-utterances-from-querylog/readme.md

@@ -23,14 +23,14 @@ The minimum prerequisites to run this sample are:
 * **[Recommended]** Visual Studio Code for IntelliSense and debugging, download it from [here](https://code.visualstudio.com/) for free.
 
 ### Install
-Install the nodeJS dependencies from NPM in the terminal/command line.
+Install the Node.js dependencies from NPM in the terminal/command line.
 
 ````
 > npm install
 ````
 
 ### Change Configuration Settings
-In order to use this application, you need to change the values in the index.js file to your own subscription key, app id, and version id. 
+In order to use this application, you need to change the values in the index.js file to your own subscription key, app ID, and version ID. 
 
 Open the index.js file, and change these values at the top of the file.
 
@@ -42,7 +42,7 @@ const LUIS_appId = "<appId>";
 const LUIS_versionId = "<versionId>";
 ````
 ### Run the application
-Run the application from a terminal/command line with nodeJs. 
+Run the application from a terminal/command line with Node.js.
 
 ````
 > node index.js
@@ -66,6 +66,12 @@ The format of the JSON for the batch upload is noted in the [batch add labels](h
 
 If you export your application data from [luis.ai applications list](https://www.luis.ai/applications) with the **Export app data to JSON file**, you will need to change the JSON format to match the stated format for the batch upload.  
 
+### Use your own private apps
+If you incorrectly use an app ID that you do not have permission to upload to, such as any public apps, you will recieve an error.
+
+### Intent And Entities are not created if NOT found
+Any intent or entity uploaded that is not found in your LUIS app will cause an error. It is important that all intents and entities used in the batch already exist in the app.
+
 ### Errors in output file of the application
 The final response body from upload api is in the 'utterances.upload.json' file. This file will be an array of responses, one response for each item in the batch.