From 708682b11a3290a58c10847c823491f170c71505 Mon Sep 17 00:00:00 2001 From: v-geberr Date: Mon, 6 Nov 2017 08:30:18 -0800 Subject: [PATCH] backup LUIS applications --- examples/backup/.gitignore | 1 + examples/backup/backup.js | 234 +++++++++++++++++++++++++++++++++++ examples/backup/package.json | 18 +++ examples/backup/readme.md | 16 +++ 4 files changed, 269 insertions(+) create mode 100644 examples/backup/.gitignore create mode 100644 examples/backup/backup.js create mode 100644 examples/backup/package.json create mode 100644 examples/backup/readme.md diff --git a/examples/backup/.gitignore b/examples/backup/.gitignore new file mode 100644 index 0000000..b512c09 --- /dev/null +++ b/examples/backup/.gitignore @@ -0,0 +1 @@ +node_modules \ No newline at end of file diff --git a/examples/backup/backup.js b/examples/backup/backup.js new file mode 100644 index 0000000..57015ba --- /dev/null +++ b/examples/backup/backup.js @@ -0,0 +1,234 @@ +// This file takes a programmatic key and returns all information for all +// applications associated with that key except for querylog. +// + +const request = require("requestretry"); +const async = require('async'); +const fs = require('fs-extra'); +const moment = require('moment'); +const path = require("path"); + +// Change to your programmatic key +const programmaticKey = "PROGRAMMATIC KEY"; + +// time delay between requests +const delayMS = 500; + +// retry recount +const retry = 5; + +// retry reqeust if error or 429 received +var retryStrategy = function (err, response, body) { + let shouldRetry = err || (response.statusCode === 429); + if (shouldRetry) console.log("retry"); + return shouldRetry; +} + +// HTTP REQUEST - get list of apps associated with programmatic key +var appListRequest = function (done) { + + let requestOptions = { + method: "GET", + url: "https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps?take=500", + headers: { + "Ocp-Apim-Subscription-Key": programmaticKey + }, + maxAttempts: retry, + retryDelay: delayMS, + retryStrategy: retryStrategy + }; + + request(requestOptions, function (err, response, body) { + if (err || response.statusCode != 200) done(); + done({ apps: JSON.parse(response.body) }); + }); +} + +// build HTTP URL list +// get list of all app infos for all apps in appList +// NOT organized by app, that will be done in fixAppList +var appListInfosUrls = (appList, done) => { + var urls = []; + + appList.apps.forEach(app => { + + console.log("app " + app.name); + + urls.push({ name: app.name, appId: app.id, "Ocp-Apim-Subscription-Key": programmaticKey, route: "appInfo", url: ("https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/" + app.id) }); + urls.push({ name: app.name, appId: app.id, "Ocp-Apim-Subscription-Key": programmaticKey, route: "versions", url: ("https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/" + app.id + "/versions?take=500") }); + urls.push({ name: app.name, appId: app.id, "Ocp-Apim-Subscription-Key": programmaticKey, route: "settings", url: ("https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/" + app.id + "/settings") }); + urls.push({ name: app.name, appId: app.id, "Ocp-Apim-Subscription-Key": programmaticKey, route: "endpoints", url: ("https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/" + app.id + "/endpoints") }); + }); + done(urls); +} + +// HTTP Request - get app info such as settings or endpoints +// result is put back into 'eachUrlObj' +var appInfoRequest = function (eachUrlObj, done) { + + console.log("request app " + eachUrlObj.url + ' ' + new moment().format("HH:mm:ss")); + + let requestOptions = { + method: "GET", + url: eachUrlObj.url, + headers: { + "Ocp-Apim-Subscription-Key": programmaticKey + }, + maxAttempts: retry, + retryDelay: delayMS, + retryStrategy: retryStrategy + }; + request(requestOptions, function (error, response, body) { + + if (error || response.statusCode != 200) done(); + console.log("response app " + response.request.href + " " + response.statusCode); + + let json = JSON.parse(body); + let route = response.request.href.substr(response.request.href.lastIndexOf('/') + 1).replace('?take=500', ''); + + let myresponse = { + appId: eachUrlObj.appId, + route: route, + name: eachUrlObj.name, + url: response.request.href, + status: response.statusCode, + body: json + }; + + // put response back into object + eachUrlObj[route] = { + response: myresponse, + status: response.statusCode + }; + + // not returning anything because response is inside eachUrlObj now + done(); + }); + +} + +// HTTP Request - get app version info for each version in app +// result is put back into 'eachUrlObj' +var appInfoVersionRequest = function (eachUrlObj, done) { + + console.log("request version " + eachUrlObj.version + ' ' + new moment().format("HH:mm:ss")); + + let requestOptions = { + method: "GET", + url: eachUrlObj.url, + headers: { + "Ocp-Apim-Subscription-Key": eachUrlObj["Ocp-Apim-Subscription-Key"] + }, + maxAttempts: retry, + retryDelay: delayMS, + retryStrategy: retryStrategy + }; + request(requestOptions, function (error, response, body) { + + console.log("response version " + response.request.href + " " + response.statusCode); + + // not returning anything because response is inside eachUrlObj now + eachUrlObj.response = JSON.parse(body); + eachUrlObj.responseStatus = response.statusCode; + done(); + }); + +} + +// build HTTP URL list to get all version infos for app +let getVersionUrlsForThisApp = (progKey, appName, appId, versionList, done) => { + let versionUrls = [] + versionList.forEach(version => { + versionUrls.push({ "Ocp-Apim-Subscription-Key": progKey, name: appName, id: appId, version: version.version, info: JSON.parse(JSON.stringify(version)), url: `https://westus.api.cognitive.microsoft.com/luis/api/v2.0/apps/${appId}/versions/${version.version}/export` }); + }); + done(versionUrls); +} + +// http request - in series +// don't call done until they are all returned +let getVersionInfos = (appUrls, versionUrls, done) => { + async.eachSeries(versionUrls, appInfoVersionRequest, (versionsResponse) => { + done(versionsResponse); + }); +} + +// put app infos children into main JSON object in right spot +var fixAppList = (appList, children) => { + + appList.forEach(app => { + var properties = children.filter(x => x.appId === app.id); + var bodies = properties.map(x => { + let info = {}; + if (x.route === "appInfo") { + info = { route: "appInfo", values: x[x.appId].response.body }; + } else { + info = { route: x.route, values: x[x.route].response.body }; + } + return info; + }); + app.properties = bodies; + }); +} + +// put version infos children into main JSON object in right spot +var fixVersionList = (appList, children) => { + appList.forEach(app => { + var versionsList = children.filter(x => x.id === app.id); + var bodies = versionsList.map(x => x.response); + app.properties.push({ route: "versionExports", values: bodies }); + }); + +} + +// main function +// appUrls is the final JSON that is tweaked as the program progresses +appListRequest((appUrls) => { + + // build app info url array + appListInfosUrls(appUrls, (appInfoUrls) => { + + // request all app infos, when all returned and done, then proceed + async.eachSeries(appInfoUrls, appInfoRequest, (response) => { + + // each info is in array but not organized by app + var appChildrenRoutes = JSON.parse(JSON.stringify(appInfoUrls)); + + // get each app's infos + var appVersionInfoResponses = appChildrenRoutes.filter(x => x.route === "versions"); + + // organize app infos back into app's spot in final JSON + fixAppList(appUrls.apps, appChildrenRoutes); + + // get version infos, an app can have more than one version + // this array will have all version info urls, not organized by app + // that organizatio will be done in fixVersionList + let versionUrls = []; + appVersionInfoResponses.forEach(app => { + let listOfVersionsForThisApp = app.versions.response.body; + getVersionUrlsForThisApp(programmaticKey, app.name, app.appId, listOfVersionsForThisApp, (appVersionUrls) => { + versionUrls.push(...appVersionUrls); + }); + }); + + // request all version infos, when all returned and done, then proceed + getVersionInfos(appUrls, versionUrls, (versionsResponse) => { + + // organize version infos back into app's spot in final JSON + fixVersionList(appUrls.apps, versionUrls); + + // write to file including programmatic key at top of JSON + fs.writeFile(path.join(__dirname, "backup.json"), JSON.stringify({ programmaticKey: programmaticKey, apps: appUrls.apps }), "utf-8") + .then(() => { + console.log("done"); + }).catch(err => { console.log(err); }); + + }); + }); + }); +}); + + + + + + diff --git a/examples/backup/package.json b/examples/backup/package.json new file mode 100644 index 0000000..261a34a --- /dev/null +++ b/examples/backup/package.json @@ -0,0 +1,18 @@ +{ + "name": "LUIS-backup", + "version": "1.0.0", + "description": "Backup all LUIS applications for given programmatic key", + "main": "backup.js", + "scripts": { + "start": "node backup.js" + }, + "license": "ISC", + "dependencies": { + "async": "^2.5.0", + "fs-extra": "^4.0.2", + "moment": "^2.19.1", + "request": "^2.83.0", + "requestretry": "^1.12.2", + "underscore": "^1.8.3" + } +} diff --git a/examples/backup/readme.md b/examples/backup/readme.md new file mode 100644 index 0000000..9df5437 --- /dev/null +++ b/examples/backup/readme.md @@ -0,0 +1,16 @@ +# Backup all apps +This Node.js file backups up all your LUIS apps: +- settings +- endpoints +- versions + +## Prerequisites +- Lastest version of Node.js. +- Install dependencies from package.json with `npm install` +- In the backup.js file, change to programmaticKey to your own key. + +## Run backup +- From command line, `npm start' or 'node backup.js' + +## Review backup +**backup.json** is a json tree structure of each app with the information about each app.