зеркало из https://github.com/microsoft/nni.git
Refactor NNI Manager UT (Stage 1) - setup toolchain (#4109)
This commit is contained in:
Родитель
2b9f5f8c35
Коммит
5fc73ba693
|
@ -165,6 +165,7 @@ stages:
|
|||
|
||||
- script: |
|
||||
python setup.py develop
|
||||
mkdir -p coverage
|
||||
displayName: Install NNI
|
||||
|
||||
- script: |
|
||||
|
@ -185,12 +186,14 @@ stages:
|
|||
python -m pytest ut/sdk/test_compressor_tf.py --cov-config=.coveragerc --cov-append
|
||||
python -m pytest ut/sdk/test_compressor_torch.py --cov-config=.coveragerc --cov-append
|
||||
python -m pytest ut/sdk/test_model_speedup.py --cov-config=.coveragerc --cov-append
|
||||
cp coverage.xml ../coverage/python.xml
|
||||
displayName: Python unit test
|
||||
|
||||
- script: |
|
||||
set -e
|
||||
cd ts/nni_manager
|
||||
yarn test
|
||||
cp coverage/cobertura-coverage.xml ../../coverage/typescript.xml
|
||||
displayName: TypeScript unit test
|
||||
|
||||
- task: PublishTestResults@2
|
||||
|
@ -203,7 +206,7 @@ stages:
|
|||
- task: PublishCodeCoverageResults@1
|
||||
inputs:
|
||||
codeCoverageTool: Cobertura
|
||||
summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/*coverage.xml'
|
||||
summaryFileLocation: coverage/*
|
||||
displayName: Publish code coverage results
|
||||
|
||||
- script: |
|
||||
|
|
|
@ -1,8 +1,9 @@
|
|||
# Build result
|
||||
dist/
|
||||
/dist
|
||||
/.tsbuildinfo
|
||||
|
||||
# node modules
|
||||
node_modules/
|
||||
/node_modules
|
||||
|
||||
# test files
|
||||
.experiment.test
|
||||
|
|
|
@ -0,0 +1,5 @@
|
|||
{
|
||||
"color": true,
|
||||
"require": "test/register.js",
|
||||
"timeout": "15s"
|
||||
}
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as ioc from 'typescript-ioc';
|
||||
|
||||
const Inject: (...args: any[]) => any = ioc.Inject;
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { ExperimentProfile, TrialJobStatistics } from './manager';
|
||||
import { TrialJobDetail, TrialJobStatus } from './trainingService';
|
||||
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
export namespace NNIErrorNames {
|
||||
export const NOT_FOUND: string = 'NOT_FOUND';
|
||||
export const INVALID_JOB_DETAIL: string = 'NO_VALID_JOB_DETAIL_FOUND';
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import assert from 'assert';
|
||||
|
||||
import { KubeflowOperator, OperatorApiVersion } from '../training_service/kubernetes/kubeflow/kubeflowConfig'
|
||||
import { KubernetesStorageKind } from '../training_service/kubernetes/kubernetesConfig';
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
abstract class ExperimentManager {
|
||||
public abstract getExperimentsInfo(): Promise<JSON>;
|
||||
public abstract setExperimentPath(newPath: string): void;
|
||||
|
|
|
@ -1,11 +1,9 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import assert from 'assert';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
|
||||
const API_ROOT_URL: string = '/api/v1/nni';
|
||||
|
||||
|
|
|
@ -1,11 +1,9 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as fs from 'fs';
|
||||
import fs from 'fs';
|
||||
import { Writable } from 'stream';
|
||||
import * as util from 'util';
|
||||
import util from 'util';
|
||||
|
||||
/* log level constants */
|
||||
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { MetricDataRecord, MetricType, TrialJobInfo } from './datastore';
|
||||
import { TrialJobStatus } from './trainingService';
|
||||
import { ExperimentConfig } from './experimentConfig';
|
||||
|
|
|
@ -1,10 +1,8 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { promisify } from 'util';
|
||||
import { runPythonScript } from './pythonScript';
|
||||
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as rx from 'rx';
|
||||
import rx from 'rx';
|
||||
import * as component from '../common/component';
|
||||
|
||||
@component.Singleton
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { spawn } from 'child_process';
|
||||
import { Logger, getLogger } from './log';
|
||||
|
||||
|
|
|
@ -1,11 +1,9 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import * as express from 'express';
|
||||
import * as http from 'http';
|
||||
import assert from 'assert';
|
||||
import express from 'express';
|
||||
import http from 'http';
|
||||
import { Deferred } from 'ts-deferred';
|
||||
import { getLogger, Logger } from './log';
|
||||
import { getBasePort } from './experimentStartupInfo';
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
// for readability
|
||||
const singleQuote = "'";
|
||||
const doubleQuote = '"';
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
interface TensorboardParams {
|
||||
trials: string;
|
||||
}
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* define TrialJobStatus
|
||||
*/
|
||||
|
|
|
@ -1,23 +1,21 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import assert from 'assert';
|
||||
import { randomBytes } from 'crypto';
|
||||
import * as cpp from 'child-process-promise';
|
||||
import * as cp from 'child_process';
|
||||
import cpp from 'child-process-promise';
|
||||
import cp from 'child_process';
|
||||
import { ChildProcess, spawn, StdioOptions } from 'child_process';
|
||||
import * as dgram from 'dgram';
|
||||
import * as fs from 'fs';
|
||||
import * as net from 'net';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import dgram from 'dgram';
|
||||
import fs from 'fs';
|
||||
import net from 'net';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import * as timersPromises from 'timers/promises';
|
||||
import * as lockfile from 'lockfile';
|
||||
import lockfile from 'lockfile';
|
||||
import { Deferred } from 'ts-deferred';
|
||||
import { Container } from 'typescript-ioc';
|
||||
import * as glob from 'glob';
|
||||
import glob from 'glob';
|
||||
|
||||
import { Database, DataStore } from './datastore';
|
||||
import { getExperimentStartupInfo, setExperimentStartupInfo } from './experimentStartupInfo';
|
||||
|
@ -243,7 +241,7 @@ function countFilesRecursively(directory: string): Promise<number> {
|
|||
const deferred: Deferred<number> = new Deferred<number>();
|
||||
|
||||
let timeoutId: NodeJS.Timer
|
||||
const delayTimeout: Promise<number> = new Promise((resolve: Function, reject: Function): void => {
|
||||
const delayTimeout: Promise<number> = new Promise((_resolve: Function, reject: Function): void => {
|
||||
// Set timeout and reject the promise once reach timeout (5 seconds)
|
||||
timeoutId = setTimeout(() => {
|
||||
reject(new Error(`Timeout: path ${directory} has too many files`));
|
||||
|
@ -375,11 +373,11 @@ function unixPathJoin(...paths: any[]): string {
|
|||
*/
|
||||
function withLockSync(func: Function, filePath: string, lockOpts: {[key: string]: any}, ...args: any): any {
|
||||
const lockName = path.join(path.dirname(filePath), path.basename(filePath) + `.lock.${process.pid}`);
|
||||
if (typeof lockOpts.stale === 'number'){
|
||||
if (typeof lockOpts['stale'] === 'number'){
|
||||
const lockPath = path.join(path.dirname(filePath), path.basename(filePath) + '.lock.*');
|
||||
const lockFileNames: string[] = glob.sync(lockPath);
|
||||
const canLock: boolean = lockFileNames.map((fileName) => {
|
||||
return fs.existsSync(fileName) && Date.now() - fs.statSync(fileName).mtimeMs < lockOpts.stale;
|
||||
return fs.existsSync(fileName) && Date.now() - fs.statSync(fileName).mtimeMs < lockOpts['stale'];
|
||||
}).filter(unexpired=>unexpired === true).length === 0;
|
||||
if (!canLock) {
|
||||
throw new Error('File has been locked.');
|
||||
|
|
|
@ -1,12 +1,10 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import assert from 'assert';
|
||||
import { ChildProcess } from 'child_process';
|
||||
import { EventEmitter } from 'events';
|
||||
import * as net from 'net';
|
||||
import net from 'net';
|
||||
import { Readable, Writable } from 'stream';
|
||||
import { NNIError } from '../common/errors';
|
||||
import { getLogger, Logger } from '../common/log';
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import assert from 'assert';
|
||||
import { Deferred } from 'ts-deferred';
|
||||
|
||||
import * as component from '../common/component';
|
||||
|
|
|
@ -1,12 +1,10 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import * as assert from 'assert';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import assert from 'assert';
|
||||
|
||||
import { getLogger, Logger } from '../common/log';
|
||||
import { isAlive, withLockSync, getExperimentsInfoPath, delay } from '../common/utils';
|
||||
|
|
|
@ -1,11 +1,9 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as cp from 'child_process';
|
||||
import * as path from 'path';
|
||||
import fs from 'fs';
|
||||
import cp from 'child_process';
|
||||
import path from 'path';
|
||||
import { ChildProcess } from 'child_process';
|
||||
|
||||
import * as component from '../common/component';
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import assert from 'assert';
|
||||
import { ChildProcess, StdioOptions } from 'child_process';
|
||||
import { Deferred } from 'ts-deferred';
|
||||
import * as component from '../common/component';
|
||||
|
@ -470,8 +468,6 @@ class NNIManager implements Manager {
|
|||
const module_ = await import('../training_service/reusable/routerTrainingService');
|
||||
return await module_.RouterTrainingService.construct(config);
|
||||
}
|
||||
|
||||
throw new Error(`Unsupported training service platform "${platform}"`);
|
||||
}
|
||||
|
||||
private setupTuner(command: string, cwd: string | undefined, mode: 'start' | 'resume', dataDirectory: string): void {
|
||||
|
|
|
@ -1,12 +1,10 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as sqlite3 from 'sqlite3';
|
||||
import assert from 'assert';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import sqlite3 from 'sqlite3';
|
||||
import { Deferred } from 'ts-deferred';
|
||||
|
||||
import {
|
||||
|
@ -202,7 +200,7 @@ class SqlDB implements Database {
|
|||
return deferred.promise;
|
||||
}
|
||||
|
||||
public storeMetricData(trialJobId: string, data: string): Promise<void> {
|
||||
public storeMetricData(_trialJobId: string, data: string): Promise<void> {
|
||||
const sql: string = 'insert into MetricData values (?,?,?,?,?,?)';
|
||||
const json: MetricDataRecord = JSON.parse(data);
|
||||
const args: any[] = [Date.now(), json.trialJobId, json.parameterId, json.type, json.sequence, JSON.stringify(json.data)];
|
||||
|
|
|
@ -1,8 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import 'app-module-path/register';
|
||||
import { Container, Scope } from 'typescript-ioc';
|
||||
|
||||
import * as fs from 'fs';
|
||||
|
@ -30,7 +29,7 @@ function initStartupInfo(
|
|||
setExperimentStartupInfo(createNew, experimentId, basePort, platform, logDirectory, experimentLogLevel, readonly, dispatcherPipe, urlprefix);
|
||||
}
|
||||
|
||||
async function initContainer(foreground: boolean, platformMode: string, logFileName?: string): Promise<void> {
|
||||
async function initContainer(foreground: boolean, _platformMode: string, logFileName?: string): Promise<void> {
|
||||
Container.bind(Manager)
|
||||
.to(NNIManager)
|
||||
.scope(Scope.Singleton);
|
||||
|
|
|
@ -4,13 +4,14 @@
|
|||
"main": "index.js",
|
||||
"scripts": {
|
||||
"build": "tsc",
|
||||
"test": "nyc --reporter=cobertura --reporter=html --reporter=text --report-dir=./htmlcov mocha -r ts-node/register -t 15000 --recursive test/**/*.test.ts --colors",
|
||||
"test": "nyc --reporter=cobertura --reporter=text mocha test/**/*.test.ts",
|
||||
"start": "node dist/main.js",
|
||||
"watch": "tsc --watch",
|
||||
"eslint": "npx eslint ./ --ext .ts"
|
||||
},
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"app-module-path": "^2.2.0",
|
||||
"azure-storage": "^2.10.4",
|
||||
"child-process-promise": "^2.2.1",
|
||||
"express": "^4.17.1",
|
||||
|
|
|
@ -1,12 +1,10 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as bodyParser from 'body-parser';
|
||||
import * as express from 'express';
|
||||
import * as httpProxy from 'http-proxy';
|
||||
import * as path from 'path';
|
||||
import bodyParser from 'body-parser';
|
||||
import express from 'express';
|
||||
import httpProxy from 'http-proxy';
|
||||
import path from 'path';
|
||||
import * as component from '../common/component';
|
||||
import { RestServer } from '../common/restServer'
|
||||
import { getLogDir } from '../common/utils';
|
||||
|
@ -50,7 +48,7 @@ export class NNIRestServer extends RestServer {
|
|||
target: 'https://netron.app'
|
||||
});
|
||||
});
|
||||
this.app.get(`${getPrefixUrl()}/*`, (req: express.Request, res: express.Response) => {
|
||||
this.app.get(`${getPrefixUrl()}/*`, (_req: express.Request, res: express.Response) => {
|
||||
res.sendFile(path.resolve('static/index.html'));
|
||||
});
|
||||
}
|
||||
|
|
|
@ -1,10 +1,8 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { Request, Response, Router } from 'express';
|
||||
import * as path from 'path';
|
||||
import path from 'path';
|
||||
|
||||
import * as component from '../common/component';
|
||||
import { DataStore, MetricDataRecord, TrialJobInfo } from '../common/datastore';
|
||||
|
@ -80,7 +78,7 @@ class NNIRestHandler {
|
|||
this.stop(router);
|
||||
|
||||
// Express-joi-validator configuration
|
||||
router.use((err: any, _req: Request, res: Response, _next: any) => {
|
||||
router.use((err: any, _req: Request, res: Response, _next: any): any => {
|
||||
if (err.isBoom) {
|
||||
this.log.error(err.output.payload);
|
||||
|
||||
|
@ -111,7 +109,7 @@ class NNIRestHandler {
|
|||
}
|
||||
|
||||
private version(router: Router): void {
|
||||
router.get('/version', async (req: Request, res: Response) => {
|
||||
router.get('/version', async (_req: Request, res: Response) => {
|
||||
const version = await getVersion();
|
||||
res.send(version);
|
||||
});
|
||||
|
@ -119,7 +117,7 @@ class NNIRestHandler {
|
|||
|
||||
// TODO add validators for request params, query, body
|
||||
private checkStatus(router: Router): void {
|
||||
router.get('/check-status', (req: Request, res: Response) => {
|
||||
router.get('/check-status', (_req: Request, res: Response) => {
|
||||
const ds: DataStore = component.get<DataStore>(DataStore);
|
||||
ds.init().then(() => {
|
||||
res.send(this.nniManager.getStatus());
|
||||
|
@ -133,7 +131,7 @@ class NNIRestHandler {
|
|||
}
|
||||
|
||||
private getExperimentProfile(router: Router): void {
|
||||
router.get('/experiment', (req: Request, res: Response) => {
|
||||
router.get('/experiment', (_req: Request, res: Response) => {
|
||||
this.nniManager.getExperimentProfile().then((profile: ExperimentProfile) => {
|
||||
res.send(profile);
|
||||
}).catch((err: Error) => {
|
||||
|
@ -144,7 +142,7 @@ class NNIRestHandler {
|
|||
|
||||
private updateExperimentProfile(router: Router): void {
|
||||
router.put('/experiment', (req: Request, res: Response) => {
|
||||
this.nniManager.updateExperimentProfile(req.body, req.query.update_type as ProfileUpdateType).then(() => {
|
||||
this.nniManager.updateExperimentProfile(req.body, req.query['update_type'] as ProfileUpdateType).then(() => {
|
||||
res.send();
|
||||
}).catch((err: Error) => {
|
||||
this.handleError(err, res);
|
||||
|
@ -163,7 +161,7 @@ class NNIRestHandler {
|
|||
}
|
||||
|
||||
private getImportedData(router: Router): void {
|
||||
router.get('/experiment/imported-data', (req: Request, res: Response) => {
|
||||
router.get('/experiment/imported-data', (_req: Request, res: Response) => {
|
||||
this.nniManager.getImportedData().then((importedData: string[]) => {
|
||||
res.send(JSON.stringify(importedData));
|
||||
}).catch((err: Error) => {
|
||||
|
@ -195,7 +193,7 @@ class NNIRestHandler {
|
|||
}
|
||||
|
||||
private getTrialJobStatistics(router: Router): void {
|
||||
router.get('/job-statistics', (req: Request, res: Response) => {
|
||||
router.get('/job-statistics', (_req: Request, res: Response) => {
|
||||
this.nniManager.getTrialJobStatistics().then((statistics: TrialJobStatistics[]) => {
|
||||
res.send(statistics);
|
||||
}).catch((err: Error) => {
|
||||
|
@ -224,7 +222,7 @@ class NNIRestHandler {
|
|||
|
||||
private listTrialJobs(router: Router): void {
|
||||
router.get('/trial-jobs', (req: Request, res: Response) => {
|
||||
this.nniManager.listTrialJobs(req.query.status as TrialJobStatus).then((jobInfos: TrialJobInfo[]) => {
|
||||
this.nniManager.listTrialJobs(req.query['status'] as TrialJobStatus).then((jobInfos: TrialJobInfo[]) => {
|
||||
jobInfos.forEach((trialJob: TrialJobInfo) => {
|
||||
this.setErrorPathForFailedJob(trialJob);
|
||||
});
|
||||
|
@ -237,7 +235,7 @@ class NNIRestHandler {
|
|||
|
||||
private getTrialJob(router: Router): void {
|
||||
router.get('/trial-jobs/:id', (req: Request, res: Response) => {
|
||||
this.nniManager.getTrialJob(req.params.id).then((jobDetail: TrialJobInfo) => {
|
||||
this.nniManager.getTrialJob(req.params['id']).then((jobDetail: TrialJobInfo) => {
|
||||
const jobInfo: TrialJobInfo = this.setErrorPathForFailedJob(jobDetail);
|
||||
res.send(jobInfo);
|
||||
}).catch((err: Error) => {
|
||||
|
@ -258,7 +256,7 @@ class NNIRestHandler {
|
|||
|
||||
private cancelTrialJob(router: Router): void {
|
||||
router.delete('/trial-jobs/:id', async (req: Request, res: Response) => {
|
||||
this.nniManager.cancelTrialJobByUser(req.params.id).then(() => {
|
||||
this.nniManager.cancelTrialJobByUser(req.params['id']).then(() => {
|
||||
res.send();
|
||||
}).catch((err: Error) => {
|
||||
this.handleError(err, res);
|
||||
|
@ -268,7 +266,7 @@ class NNIRestHandler {
|
|||
|
||||
private getMetricData(router: Router): void {
|
||||
router.get('/metric-data/:job_id*?', async (req: Request, res: Response) => {
|
||||
this.nniManager.getMetricData(req.params.job_id, req.query.type as MetricType).then((metricsData: MetricDataRecord[]) => {
|
||||
this.nniManager.getMetricData(req.params['job_id'], req.query['type'] as MetricType).then((metricsData: MetricDataRecord[]) => {
|
||||
res.send(metricsData);
|
||||
}).catch((err: Error) => {
|
||||
this.handleError(err, res);
|
||||
|
@ -278,8 +276,8 @@ class NNIRestHandler {
|
|||
|
||||
private getMetricDataByRange(router: Router): void {
|
||||
router.get('/metric-data-range/:min_seq_id/:max_seq_id', async (req: Request, res: Response) => {
|
||||
const minSeqId = Number(req.params.min_seq_id);
|
||||
const maxSeqId = Number(req.params.max_seq_id);
|
||||
const minSeqId = Number(req.params['min_seq_id']);
|
||||
const maxSeqId = Number(req.params['max_seq_id']);
|
||||
this.nniManager.getMetricDataByRange(minSeqId, maxSeqId).then((metricsData: MetricDataRecord[]) => {
|
||||
res.send(metricsData);
|
||||
}).catch((err: Error) => {
|
||||
|
@ -289,7 +287,7 @@ class NNIRestHandler {
|
|||
}
|
||||
|
||||
private getLatestMetricData(router: Router): void {
|
||||
router.get('/metric-data-latest/', async (req: Request, res: Response) => {
|
||||
router.get('/metric-data-latest/', async (_req: Request, res: Response) => {
|
||||
this.nniManager.getLatestMetricData().then((metricsData: MetricDataRecord[]) => {
|
||||
res.send(metricsData);
|
||||
}).catch((err: Error) => {
|
||||
|
@ -301,11 +299,11 @@ class NNIRestHandler {
|
|||
private getTrialFile(router: Router): void {
|
||||
router.get('/trial-file/:id/:filename', async(req: Request, res: Response) => {
|
||||
let encoding: string | null = null;
|
||||
const filename = req.params.filename;
|
||||
const filename = req.params['filename'];
|
||||
if (!filename.includes('.') || filename.match(/.*\.(txt|log)/g)) {
|
||||
encoding = 'utf8';
|
||||
}
|
||||
this.nniManager.getTrialFile(req.params.id, filename).then((content: Buffer | string) => {
|
||||
this.nniManager.getTrialFile(req.params['id'], filename).then((content: Buffer | string) => {
|
||||
const contentType = content instanceof Buffer ? 'application/octet-stream' : 'text/plain';
|
||||
res.header('Content-Type', contentType);
|
||||
if (content === '') {
|
||||
|
@ -319,7 +317,7 @@ class NNIRestHandler {
|
|||
}
|
||||
|
||||
private exportData(router: Router): void {
|
||||
router.get('/export-data', (req: Request, res: Response) => {
|
||||
router.get('/export-data', (_req: Request, res: Response) => {
|
||||
this.nniManager.exportData().then((exportedData: string) => {
|
||||
res.send(exportedData);
|
||||
}).catch((err: Error) => {
|
||||
|
@ -329,7 +327,7 @@ class NNIRestHandler {
|
|||
}
|
||||
|
||||
private getExperimentMetadata(router: Router): void {
|
||||
router.get('/experiment-metadata', (req: Request, res: Response) => {
|
||||
router.get('/experiment-metadata', (_req: Request, res: Response) => {
|
||||
Promise.all([
|
||||
this.nniManager.getExperimentProfile(),
|
||||
this.experimentsManager.getExperimentsInfo()
|
||||
|
@ -347,7 +345,7 @@ class NNIRestHandler {
|
|||
}
|
||||
|
||||
private getExperimentsInfo(router: Router): void {
|
||||
router.get('/experiments-info', (req: Request, res: Response) => {
|
||||
router.get('/experiments-info', (_req: Request, res: Response) => {
|
||||
this.experimentsManager.getExperimentsInfo().then((experimentInfo: JSON) => {
|
||||
res.send(JSON.stringify(experimentInfo));
|
||||
}).catch((err: Error) => {
|
||||
|
@ -369,7 +367,7 @@ class NNIRestHandler {
|
|||
|
||||
private getTensorboardTask(router: Router): void {
|
||||
router.get('/tensorboard/:id', (req: Request, res: Response) => {
|
||||
this.tensorboardManager.getTensorboardTask(req.params.id).then((taskDetail: TensorboardTaskInfo) => {
|
||||
this.tensorboardManager.getTensorboardTask(req.params['id']).then((taskDetail: TensorboardTaskInfo) => {
|
||||
res.send(Object.assign({}, taskDetail));
|
||||
}).catch((err: Error) => {
|
||||
this.handleError(err, res);
|
||||
|
@ -379,7 +377,7 @@ class NNIRestHandler {
|
|||
|
||||
private updateTensorboardTask(router: Router): void {
|
||||
router.put('/tensorboard/:id', (req: Request, res: Response) => {
|
||||
this.tensorboardManager.updateTensorboardTask(req.params.id).then((taskDetail: TensorboardTaskInfo) => {
|
||||
this.tensorboardManager.updateTensorboardTask(req.params['id']).then((taskDetail: TensorboardTaskInfo) => {
|
||||
res.send(Object.assign({}, taskDetail));
|
||||
}).catch((err: Error) => {
|
||||
this.handleError(err, res);
|
||||
|
@ -389,7 +387,7 @@ class NNIRestHandler {
|
|||
|
||||
private stopTensorboardTask(router: Router): void {
|
||||
router.delete('/tensorboard/:id', (req: Request, res: Response) => {
|
||||
this.tensorboardManager.stopTensorboardTask(req.params.id).then((taskDetail: TensorboardTaskInfo) => {
|
||||
this.tensorboardManager.stopTensorboardTask(req.params['id']).then((taskDetail: TensorboardTaskInfo) => {
|
||||
res.send(Object.assign({}, taskDetail));
|
||||
}).catch((err: Error) => {
|
||||
this.handleError(err, res);
|
||||
|
@ -398,7 +396,7 @@ class NNIRestHandler {
|
|||
}
|
||||
|
||||
private stopAllTensorboardTask(router: Router): void {
|
||||
router.delete('/tensorboard-tasks', (req: Request, res: Response) => {
|
||||
router.delete('/tensorboard-tasks', (_req: Request, res: Response) => {
|
||||
this.tensorboardManager.stopAllTensorboardTask().then(() => {
|
||||
res.send();
|
||||
}).catch((err: Error) => {
|
||||
|
@ -408,7 +406,7 @@ class NNIRestHandler {
|
|||
}
|
||||
|
||||
private listTensorboardTask(router: Router): void {
|
||||
router.get('/tensorboard-tasks', (req: Request, res: Response) => {
|
||||
router.get('/tensorboard-tasks', (_req: Request, res: Response) => {
|
||||
this.tensorboardManager.listTensorboardTasks().then((taskDetails: TensorboardTaskInfo[]) => {
|
||||
res.send(taskDetails);
|
||||
}).catch((err: Error) => {
|
||||
|
@ -418,7 +416,7 @@ class NNIRestHandler {
|
|||
}
|
||||
|
||||
private stop(router: Router): void {
|
||||
router.delete('/experiment', (req: Request, res: Response) => {
|
||||
router.delete('/experiment', (_req: Request, res: Response) => {
|
||||
this.nniManager.stopExperimentTopHalf().then(() => {
|
||||
res.send();
|
||||
this.nniManager.stopExperimentBottomHalf();
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
const joi = require('joi');
|
||||
|
||||
export namespace ValidationSchemas {
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
import * as assert from 'assert';
|
||||
import assert from 'assert';
|
||||
import { getIPV4Address } from '../../common/utils';
|
||||
|
||||
it('getIpv4Address', async () => {
|
||||
|
|
|
@ -43,7 +43,7 @@ function startProcess(): void {
|
|||
}
|
||||
);
|
||||
const proc: ChildProcess = getTunerProc(dispatcherCmd, stdio, 'core/test', process.env);
|
||||
proc.on('error', (error: Error): void => {
|
||||
proc.on('error', (_error: Error): void => {
|
||||
procExit = true;
|
||||
procError = true;
|
||||
});
|
||||
|
|
|
@ -169,9 +169,9 @@ describe('Unit test for nnimanager', function () {
|
|||
|
||||
it('test getTrialJob with invalid id', () => {
|
||||
//query a not exist id, and the function should throw error, and should not process then() method
|
||||
return nniManager.getTrialJob('4567').then((jobid) => {
|
||||
return nniManager.getTrialJob('4567').then((_jobid) => {
|
||||
assert.fail();
|
||||
}).catch((error) => {
|
||||
}).catch((_error) => {
|
||||
assert.isTrue(true);
|
||||
})
|
||||
})
|
||||
|
@ -252,9 +252,9 @@ describe('Unit test for nnimanager', function () {
|
|||
|
||||
it('test getMetricData with invalid trialJobId', () => {
|
||||
//query an invalid trialJobId
|
||||
return nniManager.getMetricData('43210', 'CUSTOM').then((metricData) => {
|
||||
return nniManager.getMetricData('43210', 'CUSTOM').then((_metricData) => {
|
||||
assert.fail();
|
||||
}).catch((error) => {
|
||||
}).catch((_error) => {
|
||||
})
|
||||
})
|
||||
|
||||
|
|
|
@ -204,7 +204,7 @@ class MockedDataStore implements DataStore {
|
|||
return Promise.resolve(ret);
|
||||
}
|
||||
|
||||
public getTrialJob(trialJobId: string): Promise<TrialJobInfo> {
|
||||
public getTrialJob(_trialJobId: string): Promise<TrialJobInfo> {
|
||||
return Promise.resolve({
|
||||
trialJobId: '1234',
|
||||
status: 'SUCCEEDED',
|
||||
|
|
|
@ -25,16 +25,16 @@ export class mockedeExperimentManager extends ExperimentManager {
|
|||
"logDir": null
|
||||
}
|
||||
}));
|
||||
return new Promise<JSON>((resolve, reject) => {
|
||||
return new Promise<JSON>((resolve, _reject) => {
|
||||
resolve(expInfo);
|
||||
});
|
||||
}
|
||||
|
||||
public setExperimentPath(newPath: string): void {
|
||||
public setExperimentPath(_newPath: string): void {
|
||||
return
|
||||
}
|
||||
|
||||
public setExperimentInfo(experimentId: string, key: string, value: any): void {
|
||||
public setExperimentInfo(_experimentId: string, _key: string, _value: any): void {
|
||||
return
|
||||
}
|
||||
|
||||
|
|
|
@ -27,10 +27,10 @@ export class MockedNNIManager extends Manager {
|
|||
errors: []
|
||||
}
|
||||
}
|
||||
public updateExperimentProfile(experimentProfile: ExperimentProfile, updateType: ProfileUpdateType): Promise<void> {
|
||||
public updateExperimentProfile(_experimentProfile: ExperimentProfile, _updateType: ProfileUpdateType): Promise<void> {
|
||||
return Promise.resolve();
|
||||
}
|
||||
public importData(data: string): Promise<void> {
|
||||
public importData(_data: string): Promise<void> {
|
||||
return Promise.resolve();
|
||||
}
|
||||
public getImportedData(): Promise<string[]> {
|
||||
|
@ -53,7 +53,7 @@ export class MockedNNIManager extends Manager {
|
|||
|
||||
return deferred.promise;
|
||||
}
|
||||
public addCustomizedTrialJob(hyperParams: string): Promise<number> {
|
||||
public addCustomizedTrialJob(_hyperParams: string): Promise<number> {
|
||||
return Promise.resolve(99);
|
||||
}
|
||||
|
||||
|
@ -61,7 +61,7 @@ export class MockedNNIManager extends Manager {
|
|||
return Promise.resolve();
|
||||
}
|
||||
|
||||
public submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
|
||||
public submitTrialJob(_form: TrialJobApplicationForm): Promise<TrialJobDetail> {
|
||||
const deferred: Deferred<TrialJobDetail> = new Deferred<TrialJobDetail>();
|
||||
const jobDetail: TrialJobDetail = {
|
||||
id: '1234',
|
||||
|
@ -82,23 +82,23 @@ export class MockedNNIManager extends Manager {
|
|||
return deferred.promise;
|
||||
}
|
||||
|
||||
public cancelTrialJobByUser(trialJobId: string): Promise<void> {
|
||||
public cancelTrialJobByUser(_trialJobId: string): Promise<void> {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
public getClusterMetadata(key: string): Promise<string> {
|
||||
public getClusterMetadata(_key: string): Promise<string> {
|
||||
return Promise.resolve('METAVALUE1');
|
||||
}
|
||||
|
||||
public startExperiment(experimentParams: ExperimentConfig): Promise<string> {
|
||||
public startExperiment(_experimentParams: ExperimentConfig): Promise<string> {
|
||||
return Promise.resolve('id-1234');
|
||||
}
|
||||
|
||||
public setClusterMetadata(key: string, value: string): Promise<void> {
|
||||
public setClusterMetadata(_key: string, _value: string): Promise<void> {
|
||||
return Promise.resolve();
|
||||
}
|
||||
|
||||
public getTrialJob(trialJobId: string): Promise<TrialJobInfo> {
|
||||
public getTrialJob(_trialJobId: string): Promise<TrialJobInfo> {
|
||||
const deferred: Deferred<TrialJobInfo> = new Deferred<TrialJobInfo>();
|
||||
const jobInfo: TrialJobInfo = {
|
||||
trialJobId: '1234',
|
||||
|
@ -120,16 +120,16 @@ export class MockedNNIManager extends Manager {
|
|||
public stopExperimentBottomHalf(): Promise<void> {
|
||||
throw new MethodNotImplementedError();
|
||||
}
|
||||
public getMetricData(trialJobId: string, metricType: MetricType): Promise<MetricDataRecord[]> {
|
||||
public getMetricData(_trialJobId: string, _metricType: MetricType): Promise<MetricDataRecord[]> {
|
||||
throw new MethodNotImplementedError();
|
||||
}
|
||||
public getMetricDataByRange(minSeqId: number, maxSeqId: number): Promise<MetricDataRecord[]> {
|
||||
public getMetricDataByRange(_minSeqId: number, _maxSeqId: number): Promise<MetricDataRecord[]> {
|
||||
throw new MethodNotImplementedError();
|
||||
}
|
||||
public getLatestMetricData(): Promise<MetricDataRecord[]> {
|
||||
throw new MethodNotImplementedError();
|
||||
}
|
||||
public getTrialFile(trialJobId: string, fileName: string): Promise<string> {
|
||||
public getTrialFile(_trialJobId: string, _fileName: string): Promise<string> {
|
||||
throw new MethodNotImplementedError();
|
||||
}
|
||||
public getExperimentProfile(): Promise<ExperimentProfile> {
|
||||
|
@ -161,7 +161,7 @@ export class MockedNNIManager extends Manager {
|
|||
|
||||
return Promise.resolve(profile);
|
||||
}
|
||||
public listTrialJobs(status?: TrialJobStatus): Promise<TrialJobInfo[]> {
|
||||
public listTrialJobs(_status?: TrialJobStatus): Promise<TrialJobInfo[]> {
|
||||
const job1: TrialJobInfo = {
|
||||
trialJobId: '1234',
|
||||
status: 'SUCCEEDED',
|
||||
|
|
|
@ -63,7 +63,7 @@ class MockedTrainingService extends TrainingService {
|
|||
return deferred.promise;
|
||||
}
|
||||
|
||||
public getTrialFile(trialJobId: string, fileName: string): Promise<string> {
|
||||
public getTrialFile(_trialJobId: string, _fileName: string): Promise<string> {
|
||||
throw new MethodNotImplementedError();
|
||||
}
|
||||
|
||||
|
@ -71,18 +71,18 @@ class MockedTrainingService extends TrainingService {
|
|||
|
||||
}
|
||||
|
||||
public addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
|
||||
public addTrialJobMetricListener(_listener: (_metric: TrialJobMetric) => void): void {
|
||||
}
|
||||
|
||||
public removeTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
|
||||
public removeTrialJobMetricListener(_listener: (_metric: TrialJobMetric) => void): void {
|
||||
}
|
||||
|
||||
public submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
|
||||
public submitTrialJob(_form: TrialJobApplicationForm): Promise<TrialJobDetail> {
|
||||
const deferred = new Deferred<TrialJobDetail>();
|
||||
return deferred.promise;
|
||||
}
|
||||
|
||||
public updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise<TrialJobDetail> {
|
||||
public updateTrialJob(_trialJobId: string, _form: TrialJobApplicationForm): Promise<TrialJobDetail> {
|
||||
throw new MethodNotImplementedError();
|
||||
}
|
||||
|
||||
|
@ -90,7 +90,7 @@ class MockedTrainingService extends TrainingService {
|
|||
return false;
|
||||
}
|
||||
|
||||
public cancelTrialJob(trialJobId: string, isEarlyStopped: boolean = false): Promise<void> {
|
||||
public cancelTrialJob(trialJobId: string, _isEarlyStopped: boolean = false): Promise<void> {
|
||||
const deferred = new Deferred<void>();
|
||||
if(trialJobId === '1234' || trialJobId === '3456'){
|
||||
deferred.resolve();
|
||||
|
|
|
@ -0,0 +1,2 @@
|
|||
require('ts-node/register');
|
||||
require('app-module-path/cwd');
|
|
@ -4,7 +4,7 @@
|
|||
'use strict';
|
||||
|
||||
import { assert, expect } from 'chai';
|
||||
import * as request from 'request';
|
||||
import request from 'request';
|
||||
import { Container } from 'typescript-ioc';
|
||||
|
||||
import * as component from '../../common/component';
|
||||
|
|
|
@ -3,10 +3,10 @@
|
|||
|
||||
'use strict';
|
||||
|
||||
import * as chai from 'chai';
|
||||
import * as chaiAsPromised from 'chai-as-promised';
|
||||
import * as fs from 'fs';
|
||||
import * as tmp from 'tmp';
|
||||
import chai from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
import fs from 'fs';
|
||||
import tmp from 'tmp';
|
||||
import * as component from '../../common/component';
|
||||
import { TrialJobApplicationForm, TrialJobDetail, TrainingService } from '../../common/trainingService';
|
||||
import { cleanupUnitTest, prepareUnitTest } from '../../common/utils';
|
||||
|
|
|
@ -11,7 +11,7 @@ import { execCopydir, tarAdd, validateCodeDir } from '../../training_service/com
|
|||
|
||||
const deleteFolderRecursive = (filePath: string) => {
|
||||
if (fs.existsSync(filePath)) {
|
||||
fs.readdirSync(filePath).forEach((file, index) => {
|
||||
fs.readdirSync(filePath).forEach((file, _index) => {
|
||||
const curPath = path.join(filePath, file);
|
||||
if (fs.lstatSync(curPath).isDirectory()) { // recurse
|
||||
deleteFolderRecursive(curPath);
|
||||
|
|
|
@ -3,10 +3,10 @@
|
|||
|
||||
'use strict';
|
||||
|
||||
import * as chai from 'chai';
|
||||
import * as chaiAsPromised from 'chai-as-promised';
|
||||
import * as fs from 'fs';
|
||||
import * as tmp from 'tmp';
|
||||
import chai from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
import fs from 'fs';
|
||||
import tmp from 'tmp';
|
||||
import * as component from '../../common/component';
|
||||
import { cleanupUnitTest, prepareUnitTest } from '../../common/utils';
|
||||
import { TrialConfigMetadataKey } from '../../training_service/common/trialConfigMetadataKey';
|
||||
|
|
|
@ -3,11 +3,11 @@
|
|||
|
||||
'use strict';
|
||||
|
||||
import * as chai from 'chai';
|
||||
import * as chaiAsPromised from 'chai-as-promised';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as tmp from 'tmp';
|
||||
import chai from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import tmp from 'tmp';
|
||||
import * as component from '../../common/component';
|
||||
import { TrialJobApplicationForm, TrialJobDetail} from '../../common/trainingService';
|
||||
import { cleanupUnitTest, delay, prepareUnitTest, getExperimentRootDir } from '../../common/utils';
|
||||
|
|
|
@ -3,8 +3,8 @@
|
|||
|
||||
'use strict';
|
||||
|
||||
import * as chai from 'chai';
|
||||
import * as chaiAsPromised from 'chai-as-promised';
|
||||
import chai from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
import * as component from '../../../common/component';
|
||||
import { cleanupUnitTest, prepareUnitTest } from '../../../common/utils';
|
||||
import { LinuxCommands } from '../../../training_service/remote_machine/extends/linuxCommands';
|
||||
|
|
|
@ -3,10 +3,10 @@
|
|||
|
||||
'use strict';
|
||||
|
||||
import * as cpp from 'child-process-promise';
|
||||
import * as fs from 'fs';
|
||||
import * as chai from 'chai';
|
||||
import * as chaiAsPromised from 'chai-as-promised';
|
||||
import cpp from 'child-process-promise';
|
||||
import fs from 'fs';
|
||||
import chai from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
|
||||
import { ShellExecutor } from '../../../training_service/remote_machine/shellExecutor';
|
||||
import { prepareUnitTest, cleanupUnitTest } from '../../../common/utils';
|
||||
|
|
|
@ -3,8 +3,8 @@
|
|||
|
||||
'use strict';
|
||||
|
||||
import * as chai from 'chai';
|
||||
import * as chaiAsPromised from 'chai-as-promised';
|
||||
import chai from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
import * as component from '../../../common/component';
|
||||
import { cleanupUnitTest, prepareUnitTest } from '../../../common/utils';
|
||||
import { WindowsCommands } from '../../../training_service/remote_machine/extends/windowsCommands';
|
||||
|
|
|
@ -3,11 +3,11 @@
|
|||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import * as chai from 'chai';
|
||||
import * as chaiAsPromised from 'chai-as-promised';
|
||||
import * as fs from 'fs';
|
||||
import * as tmp from 'tmp';
|
||||
import assert from 'assert';
|
||||
import chai from 'chai';
|
||||
import chaiAsPromised from 'chai-as-promised';
|
||||
import fs from 'fs';
|
||||
import tmp from 'tmp';
|
||||
import * as component from '../../common/component';
|
||||
import { TrialJobApplicationForm, TrialJobDetail, TrainingService } from '../../common/trainingService';
|
||||
import { cleanupUnitTest, delay, prepareUnitTest } from '../../common/utils';
|
||||
|
@ -133,10 +133,10 @@ describe('Unit Test for RemoteMachineTrainingService', () => {
|
|||
};
|
||||
const jobDetail: TrialJobDetail = await remoteMachineTrainingService.submitTrialJob(form);
|
||||
// Add metrics listeners
|
||||
const listener1 = function f1(metric: any) {
|
||||
const listener1 = function f1(_metric: any) {
|
||||
}
|
||||
|
||||
const listener2 = function f1(metric: any) {
|
||||
const listener2 = function f1(_metric: any) {
|
||||
}
|
||||
|
||||
remoteMachineTrainingService.addTrialJobMetricListener(listener1);
|
||||
|
|
|
@ -26,7 +26,7 @@ export class UtCommandChannel extends CommandChannel {
|
|||
return this.receivedCommands.shift();
|
||||
}
|
||||
|
||||
public async config(_key: string, value: any): Promise<void> {
|
||||
public async config(_key: string, _value: any): Promise<void> {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
|
|
|
@ -58,7 +58,7 @@ export class UtEnvironmentService extends EnvironmentService {
|
|||
// do nothing
|
||||
}
|
||||
|
||||
public async refreshEnvironmentsStatus(environments: EnvironmentInformation[]): Promise<void> {
|
||||
public async refreshEnvironmentsStatus(_environments: EnvironmentInformation[]): Promise<void> {
|
||||
// do nothing
|
||||
}
|
||||
|
||||
|
|
|
@ -1,19 +1,17 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import * as bodyParser from 'body-parser';
|
||||
import assert from 'assert';
|
||||
import bodyParser from 'body-parser';
|
||||
import { Request, Response, Router } from 'express';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { Writable } from 'stream';
|
||||
import { String } from 'typescript-string-operations';
|
||||
import * as component from '../../common/component';
|
||||
import { getBasePort, getExperimentId } from '../../common/experimentStartupInfo';
|
||||
import { RestServer } from '../../common/restServer';
|
||||
import { getExperimentRootDir, mkDirPSync } from '../../common/utils';
|
||||
import * as component from 'common/component';
|
||||
import { getBasePort, getExperimentId } from 'common/experimentStartupInfo';
|
||||
import { RestServer } from 'common/restServer';
|
||||
import { getExperimentRootDir, mkDirPSync } from 'common/utils';
|
||||
|
||||
/**
|
||||
* Cluster Job Training service Rest server, provides rest API to support Cluster job metrics update
|
||||
|
@ -108,7 +106,7 @@ export abstract class ClusterJobRestServer extends RestServer {
|
|||
|
||||
router.post(`/update-metrics/${this.expId}/:trialId`, (req: Request, res: Response) => {
|
||||
try {
|
||||
this.log.info(`Get update-metrics request, trial job id is ${req.params.trialId}`);
|
||||
this.log.info(`Get update-metrics request, trial job id is ${req.params['trialId']}`);
|
||||
this.log.info('update-metrics body is', req.body);
|
||||
|
||||
this.handleTrialMetrics(req.body.jobId, req.body.metrics);
|
||||
|
@ -127,7 +125,7 @@ export abstract class ClusterJobRestServer extends RestServer {
|
|||
this.errorMessage = `Version check failed, didn't get version check response from trialKeeper,`
|
||||
+ ` please check your NNI version in NNIManager and TrialKeeper!`;
|
||||
}
|
||||
const trialLogDir: string = path.join(getExperimentRootDir(), 'trials', req.params.trialId);
|
||||
const trialLogDir: string = path.join(getExperimentRootDir(), 'trials', req.params['trialId']);
|
||||
mkDirPSync(trialLogDir);
|
||||
const trialLogPath: string = path.join(trialLogDir, 'stdout_log_collection.log');
|
||||
try {
|
||||
|
@ -136,7 +134,7 @@ export abstract class ClusterJobRestServer extends RestServer {
|
|||
const metricsContent: any = req.body.msg.match(this.NNI_METRICS_PATTERN);
|
||||
if (metricsContent && metricsContent.groups) {
|
||||
const key: string = 'metrics';
|
||||
this.handleTrialMetrics(req.params.trialId, [metricsContent.groups[key]]);
|
||||
this.handleTrialMetrics(req.params['trialId'], [metricsContent.groups[key]]);
|
||||
skipLogging = true;
|
||||
}
|
||||
}
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
export const CONTAINER_INSTALL_NNI_SHELL_FORMAT: string =
|
||||
`#!/bin/bash
|
||||
if python3 -c 'import nni' > /dev/null 2>&1; then
|
||||
|
@ -18,4 +16,4 @@ export const CONTAINER_INSTALL_NNI_SHELL_FORMAT_FOR_WIN: string =
|
|||
if ($error -ne ''){
|
||||
python -m pip install --user --upgrade nni
|
||||
}
|
||||
exit`;
|
||||
exit`;
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
export enum ScheduleResultType {
|
||||
// Schedule succeeded
|
||||
SUCCEED,
|
||||
|
@ -65,14 +63,15 @@ export class GPUSummary {
|
|||
|
||||
|
||||
export function parseGpuIndices(gpuIndices?: string): Set<number> | undefined {
|
||||
if (gpuIndices !== undefined) {
|
||||
const indices: number[] = gpuIndices.split(',')
|
||||
.map((x: string) => parseInt(x, 10));
|
||||
if (indices.length > 0) {
|
||||
return new Set(indices);
|
||||
} else {
|
||||
throw new Error('gpuIndices can not be empty if specified.');
|
||||
}
|
||||
if (gpuIndices === undefined) {
|
||||
return undefined;
|
||||
}
|
||||
const indices: number[] = gpuIndices.split(',')
|
||||
.map((x: string) => parseInt(x, 10));
|
||||
if (indices.length > 0) {
|
||||
return new Set(indices);
|
||||
} else {
|
||||
throw new Error('gpuIndices can not be empty if specified.');
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { TrialJobStatus } from '../../common/trainingService';
|
||||
import { TrialJobStatus } from 'common/trainingService';
|
||||
|
||||
/**
|
||||
* Trial job metrics class
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Trial job configuration class
|
||||
* Representing trial job configurable properties
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Enum of metadata keys for configuration
|
||||
*/
|
||||
|
|
|
@ -1,15 +1,13 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as cpp from 'child-process-promise';
|
||||
import * as cp from 'child_process';
|
||||
import * as fs from 'fs';
|
||||
import cpp from 'child-process-promise';
|
||||
import cp from 'child_process';
|
||||
import fs from 'fs';
|
||||
import ignore from 'ignore';
|
||||
import * as path from 'path';
|
||||
import * as tar from 'tar';
|
||||
import { getLogger } from '../../common/log';
|
||||
import path from 'path';
|
||||
import tar from 'tar';
|
||||
import { getLogger } from 'common/log';
|
||||
import { String } from 'typescript-string-operations';
|
||||
import { GPU_INFO_COLLECTOR_FORMAT_WINDOWS } from './gpuData';
|
||||
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as fs from 'fs';
|
||||
import fs from 'fs';
|
||||
import { GeneralK8sClient, KubernetesCRDClient } from '../kubernetesApiClient';
|
||||
|
||||
/**
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import {KubernetesTrialConfig} from "../kubernetesConfig";
|
||||
|
||||
/**
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { AdlClientV1 } from './adlApiClient';
|
||||
import { KubernetesTrialJobDetail} from '../kubernetesData';
|
||||
import { KubernetesJobInfoCollector } from '../kubernetesJobInfoCollector';
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as component from '../../../common/component';
|
||||
import * as component from 'common/component';
|
||||
import { KubernetesJobRestServer } from '../kubernetesJobRestServer';
|
||||
import { AdlTrainingService } from './adlTrainingService';
|
||||
|
||||
|
|
|
@ -1,18 +1,16 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as component from '../../../common/component';
|
||||
import fs from 'fs';
|
||||
import * as component from 'common/component';
|
||||
|
||||
import { String } from 'typescript-string-operations';
|
||||
import { getExperimentId } from '../../../common/experimentStartupInfo';
|
||||
import { getExperimentId } from 'common/experimentStartupInfo';
|
||||
import {
|
||||
NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
|
||||
} from '../../../common/trainingService';
|
||||
import { delay, generateParamFileName, getVersion, uniqueString } from '../../../common/utils';
|
||||
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
|
||||
} from 'common/trainingService';
|
||||
import { delay, generateParamFileName, getVersion, uniqueString } from 'common/utils';
|
||||
import { TrialConfigMetadataKey } from 'training_service/common/trialConfigMetadataKey';
|
||||
import { KubernetesTrialJobDetail } from '../kubernetesData';
|
||||
import { KubernetesTrainingService } from '../kubernetesTrainingService';
|
||||
import { AdlClientFactory } from './adlApiClient'
|
||||
|
|
|
@ -1,15 +1,13 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as azureStorage from 'azure-storage';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import azureStorage from 'azure-storage';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { Deferred } from 'ts-deferred';
|
||||
import { String } from 'typescript-string-operations';
|
||||
import { getLogger } from '../../common/log';
|
||||
import { mkDirP } from '../../common/utils';
|
||||
import { getLogger } from 'common/log';
|
||||
import { mkDirP } from 'common/utils';
|
||||
|
||||
export namespace AzureStorageClientUtility {
|
||||
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as fs from 'fs';
|
||||
import fs from 'fs';
|
||||
import {GeneralK8sClient, KubernetesCRDClient} from '../kubernetesApiClient';
|
||||
|
||||
/**
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import assert from 'assert';
|
||||
|
||||
import {
|
||||
AzureStorage, KeyVaultConfig, KubernetesClusterConfig, KubernetesClusterConfigAzure, KubernetesClusterConfigNFS,
|
||||
|
@ -49,7 +47,7 @@ export class FrameworkControllerTrialConfig extends KubernetesTrialConfig {
|
|||
|
||||
export class FrameworkControllerClusterConfig extends KubernetesClusterConfig {
|
||||
public readonly serviceAccountName: string;
|
||||
constructor(apiVersion: string, serviceAccountName: string, configPath?: string, namespace?: string) {
|
||||
constructor(apiVersion: string, serviceAccountName: string, _configPath?: string, namespace?: string) {
|
||||
super(apiVersion, undefined, namespace);
|
||||
this.serviceAccountName = serviceAccountName;
|
||||
}
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { KubernetesCRDClient } from '../kubernetesApiClient';
|
||||
import { KubernetesTrialJobDetail} from '../kubernetesData';
|
||||
import { KubernetesJobInfoCollector } from '../kubernetesJobInfoCollector';
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as component from '../../../common/component';
|
||||
import * as component from 'common/component';
|
||||
import { KubernetesJobRestServer } from '../kubernetesJobRestServer';
|
||||
import { FrameworkControllerTrainingService } from './frameworkcontrollerTrainingService';
|
||||
|
||||
|
|
|
@ -1,21 +1,19 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import * as cpp from 'child-process-promise';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as component from '../../../common/component';
|
||||
import {getExperimentId} from '../../../common/experimentStartupInfo';
|
||||
import assert from 'assert';
|
||||
import cpp from 'child-process-promise';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import * as component from 'common/component';
|
||||
import {getExperimentId} from 'common/experimentStartupInfo';
|
||||
import {
|
||||
NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
|
||||
} from '../../../common/trainingService';
|
||||
import {delay, generateParamFileName, getExperimentRootDir, uniqueString} from '../../../common/utils';
|
||||
import {CONTAINER_INSTALL_NNI_SHELL_FORMAT} from '../../common/containerJobData';
|
||||
import {TrialConfigMetadataKey} from '../../common/trialConfigMetadataKey';
|
||||
import {validateCodeDir} from '../../common/util';
|
||||
} from 'common/trainingService';
|
||||
import {delay, generateParamFileName, getExperimentRootDir, uniqueString} from 'common/utils';
|
||||
import {CONTAINER_INSTALL_NNI_SHELL_FORMAT} from 'training_service/common/containerJobData';
|
||||
import {TrialConfigMetadataKey} from 'training_service/common/trialConfigMetadataKey';
|
||||
import {validateCodeDir} from 'training_service/common/util';
|
||||
import {NFSConfig} from '../kubernetesConfig';
|
||||
import {KubernetesTrialJobDetail} from '../kubernetesData';
|
||||
import {KubernetesTrainingService} from '../kubernetesTrainingService';
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as fs from 'fs';
|
||||
import fs from 'fs';
|
||||
import { GeneralK8sClient, KubernetesCRDClient } from '../kubernetesApiClient';
|
||||
import { KubeflowOperator } from './kubeflowConfig';
|
||||
|
||||
|
|
|
@ -1,10 +1,8 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import { MethodNotImplementedError } from '../../../common/errors';
|
||||
import assert from 'assert';
|
||||
import { MethodNotImplementedError } from 'common/errors';
|
||||
import { AzureStorage, KeyVaultConfig, KubernetesClusterConfig, KubernetesClusterConfigAzure, KubernetesClusterConfigNFS,
|
||||
KubernetesStorageKind, KubernetesTrialConfig, KubernetesTrialConfigTemplate, NFSConfig, StorageConfig
|
||||
} from '../kubernetesConfig';
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { KubernetesCRDClient } from '../kubernetesApiClient';
|
||||
import { KubernetesTrialJobDetail} from '../kubernetesData';
|
||||
import { KubernetesJobInfoCollector } from '../kubernetesJobInfoCollector';
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as component from '../../../common/component';
|
||||
import * as component from 'common/component';
|
||||
import { KubernetesJobRestServer } from '../kubernetesJobRestServer';
|
||||
import { KubeflowTrainingService } from './kubeflowTrainingService';
|
||||
|
||||
|
|
|
@ -1,22 +1,20 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
import assert from 'assert';
|
||||
import cpp from 'child-process-promise';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import * as component from 'common/component';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import * as cpp from 'child-process-promise';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as component from '../../../common/component';
|
||||
|
||||
import { getExperimentId } from '../../../common/experimentStartupInfo';
|
||||
import { getExperimentId } from 'common/experimentStartupInfo';
|
||||
import {
|
||||
NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
|
||||
} from '../../../common/trainingService';
|
||||
import { delay, generateParamFileName, getExperimentRootDir, uniqueString } from '../../../common/utils';
|
||||
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../../common/containerJobData';
|
||||
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
|
||||
import { validateCodeDir } from '../../common/util';
|
||||
} from 'common/trainingService';
|
||||
import { delay, generateParamFileName, getExperimentRootDir, uniqueString } from 'common/utils';
|
||||
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from 'training_service/common/containerJobData';
|
||||
import { TrialConfigMetadataKey } from 'training_service/common/trialConfigMetadataKey';
|
||||
import { validateCodeDir } from 'training_service/common/util';
|
||||
import { NFSConfig } from '../kubernetesConfig';
|
||||
import { KubernetesTrialJobDetail } from '../kubernetesData';
|
||||
import { KubernetesJobRestServer } from '../kubernetesJobRestServer';
|
||||
|
|
|
@ -1,11 +1,9 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
// eslint-disable-next-line @typescript-eslint/camelcase
|
||||
import {Client1_10, config} from 'kubernetes-client';
|
||||
import {getLogger, Logger} from '../../common/log';
|
||||
import {getLogger, Logger} from 'common/log';
|
||||
|
||||
/**
|
||||
* This function uses the environment variable
|
||||
|
|
|
@ -1,10 +1,8 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
export type KubernetesStorageKind = 'nfs' | 'azureStorage' | 'pvc';
|
||||
import {MethodNotImplementedError} from '../../common/errors';
|
||||
import {MethodNotImplementedError} from 'common/errors';
|
||||
|
||||
export abstract class KubernetesClusterConfig {
|
||||
public readonly storage?: KubernetesStorageKind;
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService';
|
||||
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from 'common/trainingService';
|
||||
|
||||
/**
|
||||
* KubeflowTrialJobDetail
|
||||
|
|
|
@ -1,12 +1,10 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import { MethodNotImplementedError, NNIError, NNIErrorNames } from '../../common/errors';
|
||||
import { getLogger, Logger } from '../../common/log';
|
||||
import { TrialJobStatus } from '../../common/trainingService';
|
||||
import assert from 'assert';
|
||||
import { MethodNotImplementedError, NNIError, NNIErrorNames } from 'common/errors';
|
||||
import { getLogger, Logger } from 'common/log';
|
||||
import { TrialJobStatus } from 'common/trainingService';
|
||||
import { KubernetesCRDClient } from './kubernetesApiClient';
|
||||
import { KubernetesTrialJobDetail } from './kubernetesData';
|
||||
|
||||
|
|
|
@ -1,10 +1,8 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { Inject } from 'typescript-ioc';
|
||||
import * as component from '../../common/component';
|
||||
import * as component from 'common/component';
|
||||
import { ClusterJobRestServer } from '../common/clusterJobRestServer';
|
||||
import { KubernetesTrainingService } from './kubernetesTrainingService';
|
||||
|
||||
|
|
|
@ -1,22 +1,20 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
import cpp from 'child-process-promise';
|
||||
import path from 'path';
|
||||
|
||||
import * as cpp from 'child-process-promise';
|
||||
import * as path from 'path';
|
||||
|
||||
import * as azureStorage from 'azure-storage';
|
||||
import azureStorage from 'azure-storage';
|
||||
import {EventEmitter} from 'events';
|
||||
import {Base64} from 'js-base64';
|
||||
import {String} from 'typescript-string-operations';
|
||||
import {getExperimentId} from '../../common/experimentStartupInfo';
|
||||
import {getLogger, Logger} from '../../common/log';
|
||||
import {MethodNotImplementedError} from '../../common/errors';
|
||||
import {getExperimentId} from 'common/experimentStartupInfo';
|
||||
import {getLogger, Logger} from 'common/log';
|
||||
import {MethodNotImplementedError} from 'common/errors';
|
||||
import {
|
||||
NNIManagerIpConfig, TrialJobDetail, TrialJobMetric
|
||||
} from '../../common/trainingService';
|
||||
import {delay, getExperimentRootDir, getIPV4Address, getJobCancelStatus, getVersion, uniqueString} from '../../common/utils';
|
||||
} from 'common/trainingService';
|
||||
import {delay, getExperimentRootDir, getIPV4Address, getJobCancelStatus, getVersion, uniqueString} from 'common/utils';
|
||||
import {AzureStorageClientUtility} from './azureStorageClientUtils';
|
||||
import {GeneralK8sClient, KubernetesCRDClient} from './kubernetesApiClient';
|
||||
import {KubernetesClusterConfig} from './kubernetesConfig';
|
||||
|
|
|
@ -1,14 +1,12 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as cpp from 'child-process-promise';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import { getLogger, Logger } from '../../common/log';
|
||||
import { delay } from '../../common/utils';
|
||||
import cpp from 'child-process-promise';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { getLogger, Logger } from 'common/log';
|
||||
import { delay } from 'common/utils';
|
||||
import { GPUInfo, GPUSummary } from '../common/gpuData';
|
||||
import { execKill, execMkdir, execRemove, execTail, runGpuMetricsCollector } from '../common/util';
|
||||
|
||||
|
|
|
@ -1,25 +1,24 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
import * as cp from 'child_process';
|
||||
import cp from 'child_process';
|
||||
import { EventEmitter } from 'events';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as ts from 'tail-stream';
|
||||
import * as tkill from 'tree-kill';
|
||||
import { NNIError, NNIErrorNames } from '../../common/errors';
|
||||
import { getExperimentId } from '../../common/experimentStartupInfo';
|
||||
import { getLogger, Logger } from '../../common/log';
|
||||
import { powershellString } from '../../common/shellUtils';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import ts from 'tail-stream';
|
||||
import tkill from 'tree-kill';
|
||||
import { NNIError, NNIErrorNames } from 'common/errors';
|
||||
import { getExperimentId } from 'common/experimentStartupInfo';
|
||||
import { getLogger, Logger } from 'common/log';
|
||||
import { powershellString } from 'common/shellUtils';
|
||||
import {
|
||||
HyperParameters, TrainingService, TrialJobApplicationForm,
|
||||
TrialJobDetail, TrialJobMetric, TrialJobStatus
|
||||
} from '../../common/trainingService';
|
||||
} from 'common/trainingService';
|
||||
import {
|
||||
delay, generateParamFileName, getExperimentRootDir, getJobCancelStatus, getNewLine, isAlive, uniqueString
|
||||
} from '../../common/utils';
|
||||
import { ExperimentConfig, LocalConfig, flattenConfig } from '../../common/experimentConfig';
|
||||
} from 'common/utils';
|
||||
import { ExperimentConfig, LocalConfig, flattenConfig } from 'common/experimentConfig';
|
||||
import { execMkdir, execNewFile, getScriptName, runScript, setEnvironmentVariable } from '../common/util';
|
||||
import { GPUScheduler } from './gpuScheduler';
|
||||
|
||||
|
@ -447,7 +446,7 @@ class LocalTrainingService implements TrainingService {
|
|||
if (process.platform !== 'win32') {
|
||||
runScriptContent.push('#!/bin/bash');
|
||||
} else {
|
||||
runScriptContent.push(`$env:PATH=${powershellString(process.env.path!)}`)
|
||||
runScriptContent.push(`$env:PATH=${powershellString(process.env['path']!)}`)
|
||||
}
|
||||
for (const variable of variables) {
|
||||
runScriptContent.push(setEnvironmentVariable(variable));
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService';
|
||||
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from 'common/trainingService';
|
||||
import {TrialConfig} from '../common/trialConfig';
|
||||
|
||||
export class PAIClusterConfig {
|
||||
|
|
|
@ -1,14 +1,12 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as request from 'request';
|
||||
import request from 'request';
|
||||
import { Deferred } from 'ts-deferred';
|
||||
import { NNIError, NNIErrorNames } from '../../common/errors';
|
||||
import { getLogger, Logger } from '../../common/log';
|
||||
import { TrialJobStatus } from '../../common/trainingService';
|
||||
import { ExperimentConfig, OpenpaiConfig } from '../../common/experimentConfig';
|
||||
import { NNIError, NNIErrorNames } from 'common/errors';
|
||||
import { getLogger, Logger } from 'common/log';
|
||||
import { TrialJobStatus } from 'common/trainingService';
|
||||
import { ExperimentConfig, OpenpaiConfig } from 'common/experimentConfig';
|
||||
import { PAITrialJobDetail } from './paiConfig';
|
||||
|
||||
interface FlattenOpenpaiConfig extends ExperimentConfig, OpenpaiConfig { }
|
||||
|
@ -44,7 +42,7 @@ export class PAIJobInfoCollector {
|
|||
await Promise.all(updatePaiTrialJobs);
|
||||
}
|
||||
|
||||
private getSinglePAITrialJobInfo(protocol: string, paiTrialJob: PAITrialJobDetail, paiToken: string, config: FlattenOpenpaiConfig): Promise<void> {
|
||||
private getSinglePAITrialJobInfo(_protocol: string, paiTrialJob: PAITrialJobDetail, paiToken: string, config: FlattenOpenpaiConfig): Promise<void> {
|
||||
const deferred: Deferred<void> = new Deferred<void>();
|
||||
if (!this.statusesNeedToCheck.includes(paiTrialJob.status)) {
|
||||
deferred.resolve();
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { Request, Response, Router } from 'express';
|
||||
import { ClusterJobRestServer } from '../common/clusterJobRestServer';
|
||||
import { PAITrainingService } from './paiTrainingService';
|
||||
|
@ -56,7 +54,7 @@ export class PAIJobRestServer extends ClusterJobRestServer {
|
|||
}
|
||||
});
|
||||
|
||||
router.get(`/parameter-file-meta`, (req: Request, res: Response) => {
|
||||
router.get(`/parameter-file-meta`, (_req: Request, res: Response) => {
|
||||
try {
|
||||
this.log.info(`GET /parameter-file-meta`);
|
||||
res.send(this.parameterFileMetaList);
|
||||
|
|
|
@ -1,29 +1,27 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import * as request from 'request';
|
||||
import * as component from '../../common/component';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import request from 'request';
|
||||
import * as component from 'common/component';
|
||||
|
||||
import { EventEmitter } from 'events';
|
||||
import { Deferred } from 'ts-deferred';
|
||||
import { getExperimentId } from '../../common/experimentStartupInfo';
|
||||
import { getLogger, Logger } from '../../common/log';
|
||||
import { MethodNotImplementedError } from '../../common/errors';
|
||||
import { getExperimentId } from 'common/experimentStartupInfo';
|
||||
import { getLogger, Logger } from 'common/log';
|
||||
import { MethodNotImplementedError } from 'common/errors';
|
||||
import {
|
||||
HyperParameters, NNIManagerIpConfig, TrainingService,
|
||||
TrialJobApplicationForm, TrialJobDetail, TrialJobMetric
|
||||
} from '../../common/trainingService';
|
||||
import { delay } from '../../common/utils';
|
||||
import { ExperimentConfig, OpenpaiConfig, flattenConfig, toMegaBytes } from '../../common/experimentConfig';
|
||||
} from 'common/trainingService';
|
||||
import { delay } from 'common/utils';
|
||||
import { ExperimentConfig, OpenpaiConfig, flattenConfig, toMegaBytes } from 'common/experimentConfig';
|
||||
import { PAIJobInfoCollector } from './paiJobInfoCollector';
|
||||
import { PAIJobRestServer } from './paiJobRestServer';
|
||||
import { PAITrialJobDetail, PAI_TRIAL_COMMAND_FORMAT } from './paiConfig';
|
||||
import { String } from 'typescript-string-operations';
|
||||
import { generateParamFileName, getIPV4Address, uniqueString } from '../../common/utils';
|
||||
import { generateParamFileName, getIPV4Address, uniqueString } from 'common/utils';
|
||||
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
|
||||
import { execMkdir, validateCodeDir, execCopydir } from '../common/util';
|
||||
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { OsCommands } from "../osCommands";
|
||||
import { RemoteCommandResult } from "../remoteMachineData";
|
||||
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { OsCommands } from "../osCommands";
|
||||
import { RemoteCommandResult } from "../remoteMachineData";
|
||||
|
||||
|
|
|
@ -1,12 +1,10 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import { getLogger, Logger } from '../../common/log';
|
||||
import { randomSelect } from '../../common/utils';
|
||||
import { RemoteMachineConfig } from '../../common/experimentConfig';
|
||||
import assert from 'assert';
|
||||
import { getLogger, Logger } from 'common/log';
|
||||
import { randomSelect } from 'common/utils';
|
||||
import { RemoteMachineConfig } from 'common/experimentConfig';
|
||||
import { GPUInfo, ScheduleResultType } from '../common/gpuData';
|
||||
import { ExecutorManager, RemoteMachineMeta, RemoteMachineScheduleResult, RemoteMachineTrialJobDetail } from './remoteMachineData';
|
||||
|
||||
|
@ -123,6 +121,7 @@ export class GPUScheduler {
|
|||
assert(false, 'gpuInfos is undefined');
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { RemoteCommandResult } from "./remoteMachineData";
|
||||
|
||||
abstract class OsCommands {
|
||||
|
|
|
@ -1,10 +1,8 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService';
|
||||
import { RemoteMachineConfig } from '../../common/experimentConfig';
|
||||
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from 'common/trainingService';
|
||||
import { RemoteMachineConfig } from 'common/experimentConfig';
|
||||
import { GPUInfo, GPUSummary, ScheduleResultType } from '../common/gpuData';
|
||||
import { ShellExecutor } from './shellExecutor';
|
||||
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { ClusterJobRestServer } from '../common/clusterJobRestServer';
|
||||
import { RemoteMachineTrainingService } from './remoteMachineTrainingService';
|
||||
|
||||
|
|
|
@ -1,28 +1,26 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import assert from 'assert';
|
||||
import { EventEmitter } from 'events';
|
||||
import * as fs from 'fs';
|
||||
import * as path from 'path';
|
||||
import fs from 'fs';
|
||||
import path from 'path';
|
||||
import { ShellExecutor } from 'training_service/remote_machine/shellExecutor';
|
||||
import { Deferred } from 'ts-deferred';
|
||||
import * as component from '../../common/component';
|
||||
import { NNIError, NNIErrorNames, MethodNotImplementedError } from '../../common/errors';
|
||||
import { getExperimentId } from '../../common/experimentStartupInfo';
|
||||
import { getLogger, Logger } from '../../common/log';
|
||||
import { ObservableTimer } from '../../common/observableTimer';
|
||||
import * as component from 'common/component';
|
||||
import { NNIError, NNIErrorNames, MethodNotImplementedError } from 'common/errors';
|
||||
import { getExperimentId } from 'common/experimentStartupInfo';
|
||||
import { getLogger, Logger } from 'common/log';
|
||||
import { ObservableTimer } from 'common/observableTimer';
|
||||
import {
|
||||
HyperParameters, TrainingService, TrialJobApplicationForm,
|
||||
TrialJobDetail, TrialJobMetric
|
||||
} from '../../common/trainingService';
|
||||
} from 'common/trainingService';
|
||||
import {
|
||||
delay, generateParamFileName, getExperimentRootDir, getIPV4Address, getJobCancelStatus,
|
||||
getVersion, uniqueString
|
||||
} from '../../common/utils';
|
||||
import { ExperimentConfig, RemoteConfig, RemoteMachineConfig, flattenConfig } from '../../common/experimentConfig';
|
||||
} from 'common/utils';
|
||||
import { ExperimentConfig, RemoteConfig, RemoteMachineConfig, flattenConfig } from 'common/experimentConfig';
|
||||
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
|
||||
import { GPUSummary, ScheduleResultType } from '../common/gpuData';
|
||||
import { execMkdir, validateCodeDir } from '../common/util';
|
||||
|
|
|
@ -1,23 +1,21 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as assert from 'assert';
|
||||
import * as fs from 'fs';
|
||||
import * as os from 'os';
|
||||
import * as path from 'path';
|
||||
import assert from 'assert';
|
||||
import fs from 'fs';
|
||||
import os from 'os';
|
||||
import path from 'path';
|
||||
import { Client, ClientChannel, ConnectConfig, SFTPWrapper } from 'ssh2';
|
||||
import * as stream from 'stream';
|
||||
import stream from 'stream';
|
||||
import { Deferred } from "ts-deferred";
|
||||
import { getLogger, Logger } from '../../common/log';
|
||||
import { uniqueString, randomInt } from '../../common/utils';
|
||||
import { getLogger, Logger } from 'common/log';
|
||||
import { uniqueString, randomInt } from 'common/utils';
|
||||
import { execRemove, tarAdd } from '../common/util';
|
||||
import { LinuxCommands } from "./extends/linuxCommands";
|
||||
import { WindowsCommands } from './extends/windowsCommands';
|
||||
import { OsCommands } from "./osCommands";
|
||||
import { RemoteCommandResult, RemoteMachineMeta } from "./remoteMachineData";
|
||||
import { NNIError, NNIErrorNames } from '../../common/errors';
|
||||
import { NNIError, NNIErrorNames } from 'common/errors';
|
||||
|
||||
class ShellExecutor {
|
||||
public name: string = "";
|
||||
|
|
|
@ -1,8 +1,6 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { Deferred } from 'ts-deferred';
|
||||
import { PythonShell } from 'python-shell';
|
||||
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { TrialConfig } from '../../common/trialConfig';
|
||||
import { TrialConfig } from 'training_service/common/trialConfig';
|
||||
import { EnvironmentInformation } from '../environment';
|
||||
import { AMLClient } from '../aml/amlClient';
|
||||
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { delay } from "../../../common/utils";
|
||||
import { delay } from "common/utils";
|
||||
import { AMLEnvironmentInformation } from '../aml/amlConfig';
|
||||
import { CommandChannel, RunnerConnection } from "../commandChannel";
|
||||
import { Channel, EnvironmentInformation } from "../environment";
|
||||
|
|
|
@ -1,10 +1,8 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import * as component from "../../../common/component";
|
||||
import { delay } from "../../../common/utils";
|
||||
import * as component from "common/component";
|
||||
import { delay } from "common/utils";
|
||||
import { CommandChannel, RunnerConnection } from "../commandChannel";
|
||||
import { Channel, EnvironmentInformation } from "../environment";
|
||||
import { StorageService } from "../storageService";
|
||||
|
|
|
@ -1,11 +1,9 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { Server as SocketServer } from "ws";
|
||||
import { getBasePort, getExperimentId } from "../../../common/experimentStartupInfo";
|
||||
import { INITIALIZED } from '../../../core/commands';
|
||||
import { getBasePort, getExperimentId } from "common/experimentStartupInfo";
|
||||
import { INITIALIZED } from 'core/commands';
|
||||
import { CommandChannel, RunnerConnection } from "../commandChannel";
|
||||
import { Channel, EnvironmentInformation } from "../environment";
|
||||
import { EventEmitter } from "events";
|
||||
|
|
|
@ -1,12 +1,10 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { EventEmitter } from "events";
|
||||
import { getLogger, Logger } from "../../common/log";
|
||||
import { TRIAL_COMMANDS } from "../../core/commands";
|
||||
import { encodeCommand } from "../../core/ipcInterface";
|
||||
import { getLogger, Logger } from "common/log";
|
||||
import { TRIAL_COMMANDS } from "core/commands";
|
||||
import { encodeCommand } from "core/ipcInterface";
|
||||
import { Channel, EnvironmentInformation } from "./environment";
|
||||
|
||||
const acceptedCommands: Set<string> = new Set<string>(TRIAL_COMMANDS);
|
||||
|
|
|
@ -1,11 +1,9 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { Deferred } from 'ts-deferred';
|
||||
import { PythonShell } from 'python-shell';
|
||||
import { getLogger, Logger } from '../../../common/log';
|
||||
import { getLogger, Logger } from 'common/log';
|
||||
|
||||
export class DlcClient {
|
||||
private log: Logger;
|
||||
|
|
|
@ -1,9 +1,7 @@
|
|||
// Copyright (c) Microsoft Corporation.
|
||||
// Licensed under the MIT license.
|
||||
|
||||
'use strict';
|
||||
|
||||
import { TrialConfig } from '../../common/trialConfig';
|
||||
import { TrialConfig } from 'training_service/common/trialConfig';
|
||||
import { EnvironmentInformation } from '../environment';
|
||||
import { DlcClient } from '../dlc/dlcClient';
|
||||
|
||||
|
|
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
Ссылка в новой задаче