backup LUIS applications

examples/backup/.gitignore

@@ -0,0 +1 @@
+node_modules

examples/backup/backup.js

@@ -0,0 +1,234 @@
+// This file takes a programmatic key and returns all information for all 
+// applications associated with that key except for querylog.
+//
+
+const request = require("requestretry");
+const async = require('async');
+const fs = require('fs-extra');
+const moment = require('moment');
+const path = require("path");
+
+// Change to your programmatic key
+const programmaticKey = "PROGRAMMATIC KEY";
+
+// time delay between requests
+const delayMS = 500;
+
+// retry recount
+const retry = 5;
+
+// retry reqeust if error or 429 received
+var retryStrategy = function (err, response, body) {
+  let shouldRetry = err || (response.statusCode === 429);
+  if (shouldRetry) console.log("retry");
+  return shouldRetry;
+}
+
+// HTTP REQUEST - get list of apps associated with programmatic key
+var appListRequest = function (done) {
+
+  let requestOptions = {
+    method: "GET",
+    url: "https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps?take=500",
+    headers: {
+      "Ocp-Apim-Subscription-Key": programmaticKey
+    },
+    maxAttempts: retry,
+    retryDelay: delayMS,
+    retryStrategy: retryStrategy
+  };
+
+  request(requestOptions, function (err, response, body) {
+    if (err || response.statusCode != 200) done();
+    done({ apps: JSON.parse(response.body) });
+  });
+}
+
+// build HTTP URL list
+// get list of all app infos for all apps in appList
+// NOT organized by app, that will be done in fixAppList
+var appListInfosUrls = (appList, done) => {
+  var urls = [];
+
+  appList.apps.forEach(app => {
+
+    console.log("app " + app.name);
+
+    urls.push({ name: app.name, appId: app.id, "Ocp-Apim-Subscription-Key": programmaticKey, route: "appInfo", url: ("https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/" + app.id) });
+    urls.push({ name: app.name, appId: app.id, "Ocp-Apim-Subscription-Key": programmaticKey, route: "versions", url: ("https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/" + app.id + "/versions?take=500") });
+    urls.push({ name: app.name, appId: app.id, "Ocp-Apim-Subscription-Key": programmaticKey, route: "settings", url: ("https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/" + app.id + "/settings") });
+    urls.push({ name: app.name, appId: app.id, "Ocp-Apim-Subscription-Key": programmaticKey, route: "endpoints", url: ("https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/" + app.id + "/endpoints") });
+  });
+  done(urls);
+}
+
+// HTTP Request - get app info such as settings or endpoints
+// result is put back into 'eachUrlObj' 
+var appInfoRequest = function (eachUrlObj, done) {
+
+  console.log("request app " + eachUrlObj.url + ' ' + new moment().format("HH:mm:ss"));
+
+  let requestOptions = {
+    method: "GET",
+    url: eachUrlObj.url,
+    headers: {
+      "Ocp-Apim-Subscription-Key": programmaticKey
+    },
+    maxAttempts: retry,
+    retryDelay: delayMS,
+    retryStrategy: retryStrategy
+  };
+  request(requestOptions, function (error, response, body) {
+
+    if (error || response.statusCode != 200) done();
+    console.log("response app " + response.request.href + " " + response.statusCode);
+
+    let json = JSON.parse(body);
+    let route = response.request.href.substr(response.request.href.lastIndexOf('/') + 1).replace('?take=500', '');
+
+    let myresponse = {
+      appId: eachUrlObj.appId,
+      route: route,
+      name: eachUrlObj.name,
+      url: response.request.href,
+      status: response.statusCode,
+      body: json
+    };
+
+    // put response back into object
+    eachUrlObj[route] = {
+      response: myresponse,
+      status: response.statusCode
+    };
+
+    // not returning anything because response is inside eachUrlObj now
+    done();
+  });
+
+}
+
+// HTTP Request - get app version info for each version in app
+// result is put back into 'eachUrlObj' 
+var appInfoVersionRequest = function (eachUrlObj, done) {
+
+  console.log("request version " + eachUrlObj.version + ' ' + new moment().format("HH:mm:ss"));
+
+  let requestOptions = {
+    method: "GET",
+    url: eachUrlObj.url,
+    headers: {
+      "Ocp-Apim-Subscription-Key": eachUrlObj["Ocp-Apim-Subscription-Key"]
+    },
+    maxAttempts: retry,
+    retryDelay: delayMS,
+    retryStrategy: retryStrategy
+  };
+  request(requestOptions, function (error, response, body) {
+
+    console.log("response version " + response.request.href + " " + response.statusCode);
+
+    // not returning anything because response is inside eachUrlObj now
+    eachUrlObj.response = JSON.parse(body);
+    eachUrlObj.responseStatus = response.statusCode;
+    done();
+  });
+
+}
+
+// build HTTP URL list to get all version infos for app
+let getVersionUrlsForThisApp = (progKey, appName, appId, versionList, done) => {
+  let versionUrls = []
+  versionList.forEach(version => {
+    versionUrls.push({ "Ocp-Apim-Subscription-Key": progKey, name: appName, id: appId, version: version.version, info: JSON.parse(JSON.stringify(version)), url: `https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/${appId}/versions/${version.version}/export` });
+  });
+  done(versionUrls);
+}
+
+// http request - in series
+// don't call done until they are all returned
+let getVersionInfos = (appUrls, versionUrls, done) => {
+  async.eachSeries(versionUrls, appInfoVersionRequest, (versionsResponse) => {
+    done(versionsResponse);
+  });
+}
+
+// put app infos children into main JSON object in right spot
+var fixAppList = (appList, children) => {
+
+  appList.forEach(app => {
+    var properties = children.filter(x => x.appId === app.id);
+    var bodies = properties.map(x => {
+      let info = {};
+      if (x.route === "appInfo") {
+        info = { route: "appInfo", values: x[x.appId].response.body };
+      } else {
+        info = { route: x.route, values: x[x.route].response.body };
+      }
+      return info;
+    });
+    app.properties = bodies;
+  });
+}
+
+// put version infos children into main JSON object in right spot
+var fixVersionList = (appList, children) => {
+  appList.forEach(app => {
+    var versionsList = children.filter(x => x.id === app.id);
+    var bodies = versionsList.map(x => x.response);
+    app.properties.push({ route: "versionExports", values: bodies });
+  });
+
+}
+
+// main function
+// appUrls is the final JSON that is tweaked as the program progresses
+appListRequest((appUrls) => {
+
+  // build app info url array
+  appListInfosUrls(appUrls, (appInfoUrls) => {
+
+    // request all app infos, when all returned and done, then proceed
+    async.eachSeries(appInfoUrls, appInfoRequest, (response) => {
+
+      // each info is in array but not organized by app
+      var appChildrenRoutes = JSON.parse(JSON.stringify(appInfoUrls));
+
+      // get each app's infos
+      var appVersionInfoResponses = appChildrenRoutes.filter(x => x.route === "versions");
+
+      // organize app infos back into app's spot in final JSON
+      fixAppList(appUrls.apps, appChildrenRoutes);
+
+      // get version infos, an app can have more than one version
+      // this array will have all version info urls, not organized by app
+      // that organizatio will be done in fixVersionList
+      let versionUrls = [];
+      appVersionInfoResponses.forEach(app => {
+        let listOfVersionsForThisApp = app.versions.response.body;
+        getVersionUrlsForThisApp(programmaticKey, app.name, app.appId, listOfVersionsForThisApp, (appVersionUrls) => {
+          versionUrls.push(...appVersionUrls);
+        });
+      });
+
+      // request all version infos, when all returned and done, then proceed
+      getVersionInfos(appUrls, versionUrls, (versionsResponse) => {
+
+        // organize version infos back into app's spot in final JSON 
+        fixVersionList(appUrls.apps, versionUrls);
+
+        // write to file including programmatic key at top of JSON
+        fs.writeFile(path.join(__dirname, "backup.json"), JSON.stringify({ programmaticKey: programmaticKey, apps: appUrls.apps }), "utf-8")
+          .then(() => {
+            console.log("done");
+          }).catch(err => { console.log(err); });
+
+      });
+    });
+  });
+});
+
+
+
+
+
+

examples/backup/package.json

@@ -0,0 +1,18 @@
+{
+  "name": "LUIS-backup",
+  "version": "1.0.0",
+  "description": "Backup all LUIS applications for given programmatic key",
+  "main": "backup.js",
+  "scripts": {
+    "start": "node backup.js"
+  },
+  "license": "ISC",
+  "dependencies": {
+    "async": "^2.5.0",
+    "fs-extra": "^4.0.2",
+    "moment": "^2.19.1",
+    "request": "^2.83.0",
+    "requestretry": "^1.12.2",
+    "underscore": "^1.8.3"
+  }
+}

examples/backup/readme.md

@@ -0,0 +1,16 @@
+# Backup all apps
+This Node.js file backups up all your LUIS apps:
+- settings
+- endpoints
+- versions
+
+## Prerequisites
+- Lastest version of Node.js. 
+- Install dependencies from package.json with `npm install`
+- In the backup.js file, change to programmaticKey to your own key.
+
+## Run backup
+- From command line, `npm start' or 'node backup.js'
+
+## Review backup
+**backup.json** is a json tree structure of each app with the information about each app.