Add master template. If non exist, inject master template to database. (#319)
* Added default master templates * Made default templates be added only if the database was new * bug fix: inject master-template to database if nonexist.
This commit is contained in:
jinlmsft
GitHub
Родитель
3d40abd233
Коммит
7ecddd4b1e
|
|
@ -30,6 +30,7 @@ src/ClusterBootstrap/deploy_backup/*
|
|||
#ASP.NET core
|
||||
src/WebUI/dotnet/.vs/**
|
||||
src/WebUI/dotnet/WebPortal/userconfig.json
|
||||
src/WebUI/dotnet/WebPortal/Master-Templates.json
|
||||
**/wwwroot/*
|
||||
**/bin/Release/*
|
||||
**/bin/Debug/*
|
||||
|
|
|
|||
|
|
@ -1546,6 +1546,8 @@ def deploy_webUI_on_node(ipAddress):
|
|||
os.system("mkdir -p ./deploy/WebUI")
|
||||
utils.render_template("./template/WebUI/userconfig.json","./deploy/WebUI/userconfig.json",config)
|
||||
os.system("cp --verbose ./deploy/WebUI/userconfig.json ../WebUI/dotnet/WebPortal/")
|
||||
os.system("cp --verbose ./template/WebUI/Master-Templates.json ./deploy/WebUI/Master-Templates.json")
|
||||
os.system("cp --verbose ./deploy/WebUI/Master-Templates.json ../WebUI/dotnet/WebPortal/Master-Templates.json")
|
||||
utils.sudo_scp(config["ssh_cert"],"./deploy/WebUI/userconfig.json","/etc/WebUI/userconfig.json", "core", webUIIP )
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
[{"Name":"Caffe training example","Username":"","Json":"{\"job_name\" : \"caffe training example - resnet18\", \"gpu_count\" : 1, \"work_path\" : \"samples\", \"image\" : \"bvlc/caffe:gpu\", \"cmd\" : \"caffe train -solver /work/caffe/solver_resnet18.prototxt\", \"port\" : \"\"}\r\n","Database":"Master"},{"Name":"Caffe-IPython","Username":"","Json":"{\"job_name\" : \"caffe-ipython\", \"image\" : \"bvlc/caffe:cpu\", \"cmd\" : \"apt-get update && apt-get install -y python-pip sudo && addgroup --force-badname --gid $$gid$$ domainusers && adduser --force-badname --home /home/$$username$$ --shell /bin/bash --uid $$uid$$ -gecos '' $$username$$ --disabled-password --gid $$gid$$ && pip install jupyter && cd /home/$$username$$ && sudo -H -u $$username$$ jupyter notebook --no-browser --port=8888 --ip=0.0.0.0 --notebook-dir=/\", \"is_interactive\" : true, \"run_as_root\" : true}\r\n","Database":"Master"},{"Name":"Caffe-SSH","Username":"","Json":"{\"job_name\" : \"caffe-ssh\", \"image\" : \"bvlc/caffe:cpu\", \"cmd\" : \"apt-get update && apt-get install -y openssh-server sudo && addgroup --force-badname --gid $$gid$$ domainusers && adduser --force-badname --home /home/$$username$$ --shell /bin/bash --uid $$uid$$ -gecos '' $$username$$ --disabled-password --gid $$gid$$ && adduser $$username$$ sudo && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && mkdir -p /root/.ssh && cat /work/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys && mkdir -p /home/$$username$$/.ssh && cat /work/.ssh/id_rsa.pub >> /home/$$username$$/.ssh/authorized_keys && cp /work/.ssh/id_rsa* /home/$$username$$/.ssh/ && chown -R $$username$$ /home/$$username$$/.ssh/ && service ssh restart && env | while read line; do if [[ $line != HOME=* ]] && [[ $line != INTERACTIVE* ]] ; then echo \\\"export $line\\\" >> /home/$$username$$/.bashrc; fi; done && sleep infinity\", \"is_interactive\" : true, \"port\" : \"22\", \"run_as_root\" : true}\r\n","Database":"Master"},{"Name":"CNTK-IPython","Username":"","Json":"{\"job_name\" : \"CNTK-ipython\", \"image\" : \"microsoft/cntk\", \"cmd\" : \"bash -c 'source \\\"/cntk/activate-cntk\\\" ; jupyter notebook --no-browser --port=8888 --ip=0.0.0.0 --notebook-dir=/ --allow-root'\", \"is_interactive\" : true, \"run_as_root\" : true}\r\n","Database":"Master"},{"Name":"Command Template Example","Username":"","Json":"{\"cmd\":\"this is an exaple command template\"}","Database":"Master"},{"Name":"MXNET-Python-CPU","Username":"","Json":"{\"job_name\" : \"mxnet-python-cpu\", \"image\" : \"tensorflow/tensorflow:latest-cpu\", \"cmd\" : \"export HOME=/job && jupyter notebook --no-browser --port=8888 --ip=0.0.0.0 --notebook-dir=/\", \"is_interactive\" : true}\r\n","Database":"Master"},{"Name":"MXNET-Python-GPU","Username":"","Json":"{\"job_name\" : \"mxnet-python\", \"gpu_count\" : 1, \"image\" : \"tensorflow/tensorflow:latest-gpu\", \"cmd\" : \"export HOME=/job && jupyter notebook --no-browser --port=8888 --ip=0.0.0.0 --notebook-dir=/\", \"is_interactive\" : true}\r\n","Database":"Master"},{"Name":"Tensorflow training example","Username":"","Json":"{\"job_name\" : \"Tensorflow training example - inception\", \"gpu_count\" : 1, \"work_path\" : \"samples\", \"image\" : \"tensorflow/tensorflow:0.12.1-gpu\", \"cmd\" : \"/work/tensorflow/models/inception/bazel-bin/inception/imagenet_train --num_gpus=1 --batch_size=32 --train_dir=/job/model --data_dir=/data/tensor\", \"port\" : \"\"}\r\n","Database":"Master"},{"Name":"Tensorflow-IPython-CPU","Username":"","Json":"{\"job_name\" : \"tensorflow-ipython-cpu\", \"image\" : \"tensorflow/tensorflow:latest\", \"cmd\" : \"export HOME=/job && jupyter notebook --no-browser --port=8888 --ip=0.0.0.0 --notebook-dir=/\", \"is_interactive\" : true}\r\n","Database":"Master"},{"Name":"Tensorflow-IPython-GPU","Username":"","Json":"{\"job_name\" : \"tensorflow-ipython\", \"gpu_count\" : 1, \"image\" : \"tensorflow/tensorflow:latest-gpu\", \"cmd\" : \"export HOME=/job && jupyter notebook --no-browser --port=8888 --ip=0.0.0.0 --notebook-dir=/\", \"is_interactive\" : true}\r\n","Database":"Master"},{"Name":"Tensorflow-SSH","Username":"","Json":"{\"job_name\" : \"tensorflow-ssh\", \"image\" : \"tensorflow/tensorflow:latest-gpu\", \"cmd\" : \"apt-get update && apt-get install -y openssh-server sudo && addgroup --force-badname --gid $$gid$$ domainusers && adduser --force-badname --home /home/$$username$$ --shell /bin/bash --uid $$uid$$ -gecos '' $$username$$ --disabled-password --gid $$gid$$ && adduser $$username$$ sudo && echo '%sudo ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers && mkdir -p /root/.ssh && cat /work/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys && mkdir -p /home/$$username$$/.ssh && cat /work/.ssh/id_rsa.pub >> /home/$$username$$/.ssh/authorized_keys && service ssh restart && sleep infinity\", \"is_interactive\" : true, \"port\" : \"22\", \"run_as_root\" : true}\r\n","Database":"Master"}, {"Name":"Test Template","Username":"","Json":"{\"image\":\"ubuntu:16.04\"}"}]
|
||||
|
|
@ -7,7 +7,7 @@ using WindowsAuth.models;
|
|||
using System.Net.Http;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
using System.Runtime.Serialization;
|
||||
|
||||
using Newtonsoft.Json;
|
||||
|
||||
// For more information on enabling Web API for empty projects, visit http://go.microsoft.com/fwlink/?LinkID=397860
|
||||
|
||||
|
|
@ -310,7 +310,7 @@ namespace WindowsAuth.Controllers
|
|||
private async Task<string> GetTemplatesAsync(string type)
|
||||
{
|
||||
string jsonString = "[";
|
||||
jsonString += "{\"Name\" : \"None\", \"Json\" : {}},";
|
||||
jsonString += "{\"Name\" : \"None\", \"Json\" : \"{}\"},";
|
||||
var master = GetTemplatesString(Startup.MasterDatabase, "Master", type);
|
||||
jsonString += await master;
|
||||
var currentCluster = HttpContext.Session.GetString("CurrentClusters");
|
||||
|
|
@ -336,7 +336,7 @@ namespace WindowsAuth.Controllers
|
|||
var t = "{";
|
||||
t += "\"Name\" : \"" + entry.Template + "\",";
|
||||
t += "\"Username\" : \"" + entry.Username + "\",";
|
||||
t += "\"Json\" : " + entry.Json + ",";
|
||||
t += "\"Json\" : " + JsonConvert.SerializeObject(entry.Json) + ",";
|
||||
t += "\"Database\" : \"" + databaseName + "\"";
|
||||
t += "},";
|
||||
templatesString += t;
|
||||
|
|
|
|||
|
|
@ -25,6 +25,8 @@ using Microsoft.IdentityModel.Clients.ActiveDirectory;
|
|||
using Microsoft.IdentityModel.Tokens;
|
||||
using Microsoft.AspNetCore.Http;
|
||||
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
namespace WindowsAuth
|
||||
{
|
||||
|
|
@ -103,7 +105,7 @@ namespace WindowsAuth
|
|||
throw new ArgumentException("There are no DLClusters in the configuration file");
|
||||
}
|
||||
Clusters = new Dictionary<string, DLCluster>();
|
||||
Database = new Dictionary<string, ClusterContext>();
|
||||
Database = new Dictionary<string, ClusterContext>();
|
||||
string defaultClusterName = null;
|
||||
foreach (var pair in clusters)
|
||||
{
|
||||
|
|
@ -181,26 +183,71 @@ namespace WindowsAuth
|
|||
optionsBuilderUsers.UseSqlServer(connectionUsers);
|
||||
var userDatabase = new ClusterContext(optionsBuilderUsers.Options);
|
||||
userDatabase.Database.EnsureCreated();
|
||||
Database[clusterName] = userDatabase;
|
||||
}
|
||||
|
||||
var templateDb = ConfigurationParser.GetConfiguration("MasterTemplates") as Dictionary<string, object>;
|
||||
var templatesMaster = new TemplateDatabase();
|
||||
templatesMaster.SQLDatabaseForTemplates = templateDb["SQLDatabaseForTemplates"] as string;
|
||||
templatesMaster.SQLHostname = templateDb["SQLHostname"] as string;
|
||||
templatesMaster.SQLPassword = templateDb["SQLPassword"] as string;
|
||||
templatesMaster.SQLUsername = templateDb["SQLUsername"] as string;
|
||||
var connectionTemplatesMaster = String.Format("Server={0};Database={1};User Id={2};Password={3}",
|
||||
templatesMaster.SQLHostname,
|
||||
templatesMaster.SQLDatabaseForTemplates,
|
||||
templatesMaster.SQLUsername,
|
||||
templatesMaster.SQLPassword);
|
||||
var optionsBuilderTemplatesMaster = new DbContextOptionsBuilder<ClusterContext>();
|
||||
optionsBuilderTemplatesMaster.UseSqlServer(connectionTemplatesMaster);
|
||||
var templateMasterDatabase = new ClusterContext(optionsBuilderTemplatesMaster.Options);
|
||||
templateMasterDatabase.Database.EnsureCreated();
|
||||
Database[clusterName] = userDatabase;
|
||||
}
|
||||
|
||||
var templateDb = ConfigurationParser.GetConfiguration("MasterTemplates") as Dictionary<string, object>;
|
||||
var templatesMaster = new TemplateDatabase();
|
||||
templatesMaster.SQLDatabaseForTemplates = templateDb["SQLDatabaseForTemplates"] as string;
|
||||
templatesMaster.SQLHostname = templateDb["SQLHostname"] as string;
|
||||
templatesMaster.SQLPassword = templateDb["SQLPassword"] as string;
|
||||
templatesMaster.SQLUsername = templateDb["SQLUsername"] as string;
|
||||
var connectionTemplatesMaster = String.Format("Server={0};Database={1};User Id={2};Password={3}",
|
||||
templatesMaster.SQLHostname,
|
||||
templatesMaster.SQLDatabaseForTemplates,
|
||||
templatesMaster.SQLUsername,
|
||||
templatesMaster.SQLPassword);
|
||||
var optionsBuilderTemplatesMaster = new DbContextOptionsBuilder<ClusterContext>();
|
||||
optionsBuilderTemplatesMaster.UseSqlServer(connectionTemplatesMaster);
|
||||
var templateMasterDatabase = new ClusterContext(optionsBuilderTemplatesMaster.Options);
|
||||
var created = templateMasterDatabase.Database.EnsureCreated();
|
||||
var entryArries = templateMasterDatabase.Template.Select( x => x.Template ).ToArray();
|
||||
var dic = new Dictionary<string, bool>(StringComparer.OrdinalIgnoreCase);
|
||||
foreach (var entry in entryArries)
|
||||
{
|
||||
dic.Add(entry, true);
|
||||
}
|
||||
var nEntries = entryArries.Length;
|
||||
_logger.LogInformation("# of entries in Master-Templates: {0}", nEntries);
|
||||
MasterDatabase = templateMasterDatabase;
|
||||
|
||||
var template_file = "./Master-Templates.json";
|
||||
|
||||
if ( File.Exists(template_file))
|
||||
{
|
||||
int ncount = 0;
|
||||
_logger.LogInformation("Entries in template file: {0}", template_file);
|
||||
var list = new List<Tuple<string, string>>();
|
||||
using (var file = File.OpenText(template_file))
|
||||
using (var reader = new JsonTextReader(file))
|
||||
{
|
||||
foreach (var templateTok in (JArray)JToken.ReadFrom(reader))
|
||||
{
|
||||
var template = (JObject)templateTok;
|
||||
var TName = template["Name"].Value<string>();
|
||||
var TJson = template["Json"].Value<string>();
|
||||
_logger.LogInformation("{0}: {1}, {2}", ncount, TName, TJson);
|
||||
list.Add(new Tuple< string, string>(TName, TJson));
|
||||
ncount++;
|
||||
// var sql = @"INSERT INTO dbo.Template (Template, Json, Type) VALUES ({0}, {1}, job)";
|
||||
// MasterDatabase.Database.ExecuteSqlCommand(sql, TName, TJson);
|
||||
}
|
||||
}
|
||||
if (ncount > nEntries)
|
||||
{
|
||||
// Trigger ingestion logic
|
||||
foreach (var entry in list)
|
||||
{
|
||||
if (!dic.ContainsKey(entry.Item1))
|
||||
{
|
||||
TemplateEntry entryAdd = new TemplateEntry(entry.Item1, null, entry.Item2, "job");
|
||||
MasterDatabase.Template.Add(entryAdd);
|
||||
}
|
||||
}
|
||||
MasterDatabase.SaveChanges();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
if (String.IsNullOrEmpty(defaultClusterName))
|
||||
defaultClusterName = Clusters.Keys.First<string>();
|
||||
|
|
@ -232,13 +279,13 @@ namespace WindowsAuth
|
|||
foreach (var pair in authentication)
|
||||
{
|
||||
bool bUse = (numDeployedAuthentication == 0 || deployAuthentication.ContainsKey(pair.Key));
|
||||
if ( bUse )
|
||||
if ( bUse )
|
||||
{
|
||||
var authenticationScheme = pair.Key;
|
||||
var authenticationConfig = pair.Value;
|
||||
var openIDOpt = new OpenIDAuthentication(authenticationScheme, authenticationConfig, loggerFactory);
|
||||
AuthenticationSchemes[authenticationScheme] = openIDOpt;
|
||||
app.UseOpenIdConnectAuthentication(openIDOpt);
|
||||
app.UseOpenIdConnectAuthentication(openIDOpt);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -205,9 +205,9 @@
|
|||
return template;
|
||||
}
|
||||
|
||||
function UseTemplate(tempalteValueString) {
|
||||
var templateValue = JSON.parse(tempalteValueString);
|
||||
var settings = templateValue.Json;
|
||||
function UseTemplate(templateValueString) {
|
||||
var templateValue = JSON.parse(templateValueString);
|
||||
var settings = JSON.parse(templateValue.Json);
|
||||
var master = templateValue.Database === "Master";
|
||||
document.getElementsByName("templatelocation")[0].checked = !(templateValue.Database === undefined) && master;
|
||||
document.getElementsByName("templatelocation")[1].checked = !(templateValue.Database === undefined) && !master;
|
||||
|
|
@ -331,7 +331,7 @@
|
|||
var params = {};
|
||||
params.Name = templateArray[i].Name;
|
||||
params.Username = templateArray[i].Username;
|
||||
params.Json = JSON.stringify(templateArray[i].Json);
|
||||
params.Json = templateArray[i].Json;
|
||||
var e = document.getElementById("databaseselect");
|
||||
params.Database = e.options[e.selectedIndex].value;
|
||||
SaveTemplateAPI(params);
|
||||
|
|
@ -544,4 +544,4 @@
|
|||
|
||||
<div style="height: 0px; overflow: hidden">
|
||||
<input type="file" id="fileInput" name="fileInput" onchange="UploadDatabase()"/>
|
||||
</div>
|
||||
</div>
|
||||
|
|
|
|||
Загрузка…
Ссылка в новой задаче