microsoft
/
nni
зеркало из https://github.com/microsoft/nni.git
1
0
Форкнуть 0
Код Задачи Пакеты Проекты Релизы Вики Активность

Refactor NNI Manager UT (Stage 1) - setup toolchain (#4109)

This commit is contained in:
liuzhe-lz 2021-08-27 16:48:53 +08:00 коммит произвёл GitHub
Родитель 2b9f5f8c35
Коммит 5fc73ba693
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
121 изменённых файлов: 454 добавлений и 610 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

5
pipelines/fast-test.yml
Просмотреть файл

@ -165,6 +165,7 @@ stages:
- script: |
python setup.py develop
mkdir -p coverage
displayName: Install NNI
- script: |
@ -185,12 +186,14 @@ stages:
python -m pytest ut/sdk/test_compressor_tf.py --cov-config=.coveragerc --cov-append
python -m pytest ut/sdk/test_compressor_torch.py --cov-config=.coveragerc --cov-append
python -m pytest ut/sdk/test_model_speedup.py --cov-config=.coveragerc --cov-append
cp coverage.xml ../coverage/python.xml
displayName: Python unit test
- script: |
set -e
cd ts/nni_manager
yarn test
cp coverage/cobertura-coverage.xml ../../coverage/typescript.xml
displayName: TypeScript unit test
- task: PublishTestResults@2
@ -203,7 +206,7 @@ stages:
- task: PublishCodeCoverageResults@1
inputs:
codeCoverageTool: Cobertura
summaryFileLocation: '$(System.DefaultWorkingDirectory)/**/*coverage.xml'
summaryFileLocation: coverage/*
displayName: Publish code coverage results
- script: |

5
ts/nni_manager/.gitignore поставляемый
Просмотреть файл

@ -1,8 +1,9 @@
# Build result
dist/
/dist
/.tsbuildinfo
# node modules
node_modules/
/node_modules
# test files
.experiment.test

5
ts/nni_manager/.mocharc.json Normal file
Просмотреть файл

@ -0,0 +1,5 @@
{
"color": true,
"require": "test/register.js",
"timeout": "15s"
}

2
ts/nni_manager/common/component.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as ioc from 'typescript-ioc';
const Inject: (...args: any[]) => any = ioc.Inject;

2
ts/nni_manager/common/datastore.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { ExperimentProfile, TrialJobStatistics } from './manager';
import { TrialJobDetail, TrialJobStatus } from './trainingService';

2
ts/nni_manager/common/errors.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
export namespace NNIErrorNames {
export const NOT_FOUND: string = 'NOT_FOUND';
export const INVALID_JOB_DETAIL: string = 'NO_VALID_JOB_DETAIL_FOUND';

4
ts/nni_manager/common/experimentConfig.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import { KubeflowOperator, OperatorApiVersion } from '../training_service/kubernetes/kubeflow/kubeflowConfig'
import { KubernetesStorageKind } from '../training_service/kubernetes/kubernetesConfig';

2
ts/nni_manager/common/experimentManager.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
abstract class ExperimentManager {
public abstract getExperimentsInfo(): Promise<JSON>;
public abstract setExperimentPath(newPath: string): void;

8
ts/nni_manager/common/experimentStartupInfo.ts
Просмотреть файл

@ -1,11 +1,9 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import * as os from 'os';
import * as path from 'path';
import assert from 'assert';
import os from 'os';
import path from 'path';
const API_ROOT_URL: string = '/api/v1/nni';

6
ts/nni_manager/common/log.ts
Просмотреть файл

@ -1,11 +1,9 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import fs from 'fs';
import { Writable } from 'stream';
import * as util from 'util';
import util from 'util';
/* log level constants */

2
ts/nni_manager/common/manager.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { MetricDataRecord, MetricType, TrialJobInfo } from './datastore';
import { TrialJobStatus } from './trainingService';
import { ExperimentConfig } from './experimentConfig';

6
ts/nni_manager/common/nniConfig.ts
Просмотреть файл

@ -1,10 +1,8 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import * as path from 'path';
import fs from 'fs';
import path from 'path';
import { promisify } from 'util';
import { runPythonScript } from './pythonScript';

4
ts/nni_manager/common/observableTimer.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as rx from 'rx';
import rx from 'rx';
import * as component from '../common/component';
@component.Singleton

2
ts/nni_manager/common/pythonScript.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { spawn } from 'child_process';
import { Logger, getLogger } from './log';

8
ts/nni_manager/common/restServer.ts
Просмотреть файл

@ -1,11 +1,9 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import * as express from 'express';
import * as http from 'http';
import assert from 'assert';
import express from 'express';
import http from 'http';
import { Deferred } from 'ts-deferred';
import { getLogger, Logger } from './log';
import { getBasePort } from './experimentStartupInfo';

2
ts/nni_manager/common/shellUtils.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
// for readability
const singleQuote = "'";
const doubleQuote = '"';

2
ts/nni_manager/common/tensorboardManager.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
interface TensorboardParams {
trials: string;
}

2
ts/nni_manager/common/trainingService.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
/**
* define TrialJobStatus
*/

28
ts/nni_manager/common/utils.ts
Просмотреть файл

@ -1,23 +1,21 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import { randomBytes } from 'crypto';
import * as cpp from 'child-process-promise';
import * as cp from 'child_process';
import cpp from 'child-process-promise';
import cp from 'child_process';
import { ChildProcess, spawn, StdioOptions } from 'child_process';
import * as dgram from 'dgram';
import * as fs from 'fs';
import * as net from 'net';
import * as os from 'os';
import * as path from 'path';
import dgram from 'dgram';
import fs from 'fs';
import net from 'net';
import os from 'os';
import path from 'path';
import * as timersPromises from 'timers/promises';
import * as lockfile from 'lockfile';
import lockfile from 'lockfile';
import { Deferred } from 'ts-deferred';
import { Container } from 'typescript-ioc';
import * as glob from 'glob';
import glob from 'glob';
import { Database, DataStore } from './datastore';
import { getExperimentStartupInfo, setExperimentStartupInfo } from './experimentStartupInfo';
@ -243,7 +241,7 @@ function countFilesRecursively(directory: string): Promise<number> {
const deferred: Deferred<number> = new Deferred<number>();
let timeoutId: NodeJS.Timer
const delayTimeout: Promise<number> = new Promise((resolve: Function, reject: Function): void => {
const delayTimeout: Promise<number> = new Promise((_resolve: Function, reject: Function): void => {
// Set timeout and reject the promise once reach timeout (5 seconds)
timeoutId = setTimeout(() => {
reject(new Error(`Timeout: path ${directory} has too many files`));
@ -375,11 +373,11 @@ function unixPathJoin(...paths: any[]): string {
*/
function withLockSync(func: Function, filePath: string, lockOpts: {[key: string]: any}, ...args: any): any {
const lockName = path.join(path.dirname(filePath), path.basename(filePath) + `.lock.${process.pid}`);
if (typeof lockOpts.stale === 'number'){
if (typeof lockOpts['stale'] === 'number'){
const lockPath = path.join(path.dirname(filePath), path.basename(filePath) + '.lock.*');
const lockFileNames: string[] = glob.sync(lockPath);
const canLock: boolean = lockFileNames.map((fileName) => {
return fs.existsSync(fileName) && Date.now() - fs.statSync(fileName).mtimeMs < lockOpts.stale;
return fs.existsSync(fileName) && Date.now() - fs.statSync(fileName).mtimeMs < lockOpts['stale'];
}).filter(unexpired=>unexpired === true).length === 0;
if (!canLock) {
throw new Error('File has been locked.');

6
ts/nni_manager/core/ipcInterface.ts
Просмотреть файл

@ -1,12 +1,10 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import { ChildProcess } from 'child_process';
import { EventEmitter } from 'events';
import * as net from 'net';
import net from 'net';
import { Readable, Writable } from 'stream';
import { NNIError } from '../common/errors';
import { getLogger, Logger } from '../common/log';

4
ts/nni_manager/core/nniDataStore.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import { Deferred } from 'ts-deferred';
import * as component from '../common/component';

10
ts/nni_manager/core/nniExperimentsManager.ts
Просмотреть файл

@ -1,12 +1,10 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import * as assert from 'assert';
import fs from 'fs';
import os from 'os';
import path from 'path';
import assert from 'assert';
import { getLogger, Logger } from '../common/log';
import { isAlive, withLockSync, getExperimentsInfoPath, delay } from '../common/utils';

8
ts/nni_manager/core/nniTensorboardManager.ts
Просмотреть файл

@ -1,11 +1,9 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import * as cp from 'child_process';
import * as path from 'path';
import fs from 'fs';
import cp from 'child_process';
import path from 'path';
import { ChildProcess } from 'child_process';
import * as component from '../common/component';

6
ts/nni_manager/core/nnimanager.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import { ChildProcess, StdioOptions } from 'child_process';
import { Deferred } from 'ts-deferred';
import * as component from '../common/component';
@ -470,8 +468,6 @@ class NNIManager implements Manager {
const module_ = await import('../training_service/reusable/routerTrainingService');
return await module_.RouterTrainingService.construct(config);
}
throw new Error(`Unsupported training service platform "${platform}"`);
}
private setupTuner(command: string, cwd: string | undefined, mode: 'start' | 'resume', dataDirectory: string): void {

12
ts/nni_manager/core/sqlDatabase.ts
Просмотреть файл

@ -1,12 +1,10 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import * as fs from 'fs';
import * as path from 'path';
import * as sqlite3 from 'sqlite3';
import assert from 'assert';
import fs from 'fs';
import path from 'path';
import sqlite3 from 'sqlite3';
import { Deferred } from 'ts-deferred';
import {
@ -202,7 +200,7 @@ class SqlDB implements Database {
return deferred.promise;
}
public storeMetricData(trialJobId: string, data: string): Promise<void> {
public storeMetricData(_trialJobId: string, data: string): Promise<void> {
const sql: string = 'insert into MetricData values (?,?,?,?,?,?)';
const json: MetricDataRecord = JSON.parse(data);
const args: any[] = [Date.now(), json.trialJobId, json.parameterId, json.type, json.sequence, JSON.stringify(json.data)];

5
ts/nni_manager/main.ts
Просмотреть файл

@ -1,8 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import 'app-module-path/register';
import { Container, Scope } from 'typescript-ioc';
import * as fs from 'fs';
@ -30,7 +29,7 @@ function initStartupInfo(
setExperimentStartupInfo(createNew, experimentId, basePort, platform, logDirectory, experimentLogLevel, readonly, dispatcherPipe, urlprefix);
}
async function initContainer(foreground: boolean, platformMode: string, logFileName?: string): Promise<void> {
async function initContainer(foreground: boolean, _platformMode: string, logFileName?: string): Promise<void> {
Container.bind(Manager)
.to(NNIManager)
.scope(Scope.Singleton);

3
ts/nni_manager/package.json
Просмотреть файл

@ -4,13 +4,14 @@
"main": "index.js",
"scripts": {
"build": "tsc",
"test": "nyc --reporter=cobertura --reporter=html --reporter=text --report-dir=./htmlcov mocha -r ts-node/register -t 15000 --recursive test/**/*.test.ts --colors",
"test": "nyc --reporter=cobertura --reporter=text mocha test/**/*.test.ts",
"start": "node dist/main.js",
"watch": "tsc --watch",
"eslint": "npx eslint ./ --ext .ts"
},
"license": "MIT",
"dependencies": {
"app-module-path": "^2.2.0",
"azure-storage": "^2.10.4",
"child-process-promise": "^2.2.1",
"express": "^4.17.1",

12
ts/nni_manager/rest_server/nniRestServer.ts
Просмотреть файл

@ -1,12 +1,10 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as bodyParser from 'body-parser';
import * as express from 'express';
import * as httpProxy from 'http-proxy';
import * as path from 'path';
import bodyParser from 'body-parser';
import express from 'express';
import httpProxy from 'http-proxy';
import path from 'path';
import * as component from '../common/component';
import { RestServer } from '../common/restServer'
import { getLogDir } from '../common/utils';
@ -50,7 +48,7 @@ export class NNIRestServer extends RestServer {
target: 'https://netron.app'
});
});
this.app.get(`${getPrefixUrl()}/*`, (req: express.Request, res: express.Response) => {
this.app.get(`${getPrefixUrl()}/*`, (_req: express.Request, res: express.Response) => {
res.sendFile(path.resolve('static/index.html'));
});
}

54
ts/nni_manager/rest_server/restHandler.ts
Просмотреть файл

@ -1,10 +1,8 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { Request, Response, Router } from 'express';
import * as path from 'path';
import path from 'path';
import * as component from '../common/component';
import { DataStore, MetricDataRecord, TrialJobInfo } from '../common/datastore';
@ -80,7 +78,7 @@ class NNIRestHandler {
this.stop(router);
// Express-joi-validator configuration
router.use((err: any, _req: Request, res: Response, _next: any) => {
router.use((err: any, _req: Request, res: Response, _next: any): any => {
if (err.isBoom) {
this.log.error(err.output.payload);
@ -111,7 +109,7 @@ class NNIRestHandler {
}
private version(router: Router): void {
router.get('/version', async (req: Request, res: Response) => {
router.get('/version', async (_req: Request, res: Response) => {
const version = await getVersion();
res.send(version);
});
@ -119,7 +117,7 @@ class NNIRestHandler {
// TODO add validators for request params, query, body
private checkStatus(router: Router): void {
router.get('/check-status', (req: Request, res: Response) => {
router.get('/check-status', (_req: Request, res: Response) => {
const ds: DataStore = component.get<DataStore>(DataStore);
ds.init().then(() => {
res.send(this.nniManager.getStatus());
@ -133,7 +131,7 @@ class NNIRestHandler {
}
private getExperimentProfile(router: Router): void {
router.get('/experiment', (req: Request, res: Response) => {
router.get('/experiment', (_req: Request, res: Response) => {
this.nniManager.getExperimentProfile().then((profile: ExperimentProfile) => {
res.send(profile);
}).catch((err: Error) => {
@ -144,7 +142,7 @@ class NNIRestHandler {
private updateExperimentProfile(router: Router): void {
router.put('/experiment', (req: Request, res: Response) => {
this.nniManager.updateExperimentProfile(req.body, req.query.update_type as ProfileUpdateType).then(() => {
this.nniManager.updateExperimentProfile(req.body, req.query['update_type'] as ProfileUpdateType).then(() => {
res.send();
}).catch((err: Error) => {
this.handleError(err, res);
@ -163,7 +161,7 @@ class NNIRestHandler {
}
private getImportedData(router: Router): void {
router.get('/experiment/imported-data', (req: Request, res: Response) => {
router.get('/experiment/imported-data', (_req: Request, res: Response) => {
this.nniManager.getImportedData().then((importedData: string[]) => {
res.send(JSON.stringify(importedData));
}).catch((err: Error) => {
@ -195,7 +193,7 @@ class NNIRestHandler {
}
private getTrialJobStatistics(router: Router): void {
router.get('/job-statistics', (req: Request, res: Response) => {
router.get('/job-statistics', (_req: Request, res: Response) => {
this.nniManager.getTrialJobStatistics().then((statistics: TrialJobStatistics[]) => {
res.send(statistics);
}).catch((err: Error) => {
@ -224,7 +222,7 @@ class NNIRestHandler {
private listTrialJobs(router: Router): void {
router.get('/trial-jobs', (req: Request, res: Response) => {
this.nniManager.listTrialJobs(req.query.status as TrialJobStatus).then((jobInfos: TrialJobInfo[]) => {
this.nniManager.listTrialJobs(req.query['status'] as TrialJobStatus).then((jobInfos: TrialJobInfo[]) => {
jobInfos.forEach((trialJob: TrialJobInfo) => {
this.setErrorPathForFailedJob(trialJob);
});
@ -237,7 +235,7 @@ class NNIRestHandler {
private getTrialJob(router: Router): void {
router.get('/trial-jobs/:id', (req: Request, res: Response) => {
this.nniManager.getTrialJob(req.params.id).then((jobDetail: TrialJobInfo) => {
this.nniManager.getTrialJob(req.params['id']).then((jobDetail: TrialJobInfo) => {
const jobInfo: TrialJobInfo = this.setErrorPathForFailedJob(jobDetail);
res.send(jobInfo);
}).catch((err: Error) => {
@ -258,7 +256,7 @@ class NNIRestHandler {
private cancelTrialJob(router: Router): void {
router.delete('/trial-jobs/:id', async (req: Request, res: Response) => {
this.nniManager.cancelTrialJobByUser(req.params.id).then(() => {
this.nniManager.cancelTrialJobByUser(req.params['id']).then(() => {
res.send();
}).catch((err: Error) => {
this.handleError(err, res);
@ -268,7 +266,7 @@ class NNIRestHandler {
private getMetricData(router: Router): void {
router.get('/metric-data/:job_id*?', async (req: Request, res: Response) => {
this.nniManager.getMetricData(req.params.job_id, req.query.type as MetricType).then((metricsData: MetricDataRecord[]) => {
this.nniManager.getMetricData(req.params['job_id'], req.query['type'] as MetricType).then((metricsData: MetricDataRecord[]) => {
res.send(metricsData);
}).catch((err: Error) => {
this.handleError(err, res);
@ -278,8 +276,8 @@ class NNIRestHandler {
private getMetricDataByRange(router: Router): void {
router.get('/metric-data-range/:min_seq_id/:max_seq_id', async (req: Request, res: Response) => {
const minSeqId = Number(req.params.min_seq_id);
const maxSeqId = Number(req.params.max_seq_id);
const minSeqId = Number(req.params['min_seq_id']);
const maxSeqId = Number(req.params['max_seq_id']);
this.nniManager.getMetricDataByRange(minSeqId, maxSeqId).then((metricsData: MetricDataRecord[]) => {
res.send(metricsData);
}).catch((err: Error) => {
@ -289,7 +287,7 @@ class NNIRestHandler {
}
private getLatestMetricData(router: Router): void {
router.get('/metric-data-latest/', async (req: Request, res: Response) => {
router.get('/metric-data-latest/', async (_req: Request, res: Response) => {
this.nniManager.getLatestMetricData().then((metricsData: MetricDataRecord[]) => {
res.send(metricsData);
}).catch((err: Error) => {
@ -301,11 +299,11 @@ class NNIRestHandler {
private getTrialFile(router: Router): void {
router.get('/trial-file/:id/:filename', async(req: Request, res: Response) => {
let encoding: string | null = null;
const filename = req.params.filename;
const filename = req.params['filename'];
if (!filename.includes('.') || filename.match(/.*\.(txt|log)/g)) {
encoding = 'utf8';
}
this.nniManager.getTrialFile(req.params.id, filename).then((content: Buffer | string) => {
this.nniManager.getTrialFile(req.params['id'], filename).then((content: Buffer | string) => {
const contentType = content instanceof Buffer ? 'application/octet-stream' : 'text/plain';
res.header('Content-Type', contentType);
if (content === '') {
@ -319,7 +317,7 @@ class NNIRestHandler {
}
private exportData(router: Router): void {
router.get('/export-data', (req: Request, res: Response) => {
router.get('/export-data', (_req: Request, res: Response) => {
this.nniManager.exportData().then((exportedData: string) => {
res.send(exportedData);
}).catch((err: Error) => {
@ -329,7 +327,7 @@ class NNIRestHandler {
}
private getExperimentMetadata(router: Router): void {
router.get('/experiment-metadata', (req: Request, res: Response) => {
router.get('/experiment-metadata', (_req: Request, res: Response) => {
Promise.all([
this.nniManager.getExperimentProfile(),
this.experimentsManager.getExperimentsInfo()
@ -347,7 +345,7 @@ class NNIRestHandler {
}
private getExperimentsInfo(router: Router): void {
router.get('/experiments-info', (req: Request, res: Response) => {
router.get('/experiments-info', (_req: Request, res: Response) => {
this.experimentsManager.getExperimentsInfo().then((experimentInfo: JSON) => {
res.send(JSON.stringify(experimentInfo));
}).catch((err: Error) => {
@ -369,7 +367,7 @@ class NNIRestHandler {
private getTensorboardTask(router: Router): void {
router.get('/tensorboard/:id', (req: Request, res: Response) => {
this.tensorboardManager.getTensorboardTask(req.params.id).then((taskDetail: TensorboardTaskInfo) => {
this.tensorboardManager.getTensorboardTask(req.params['id']).then((taskDetail: TensorboardTaskInfo) => {
res.send(Object.assign({}, taskDetail));
}).catch((err: Error) => {
this.handleError(err, res);
@ -379,7 +377,7 @@ class NNIRestHandler {
private updateTensorboardTask(router: Router): void {
router.put('/tensorboard/:id', (req: Request, res: Response) => {
this.tensorboardManager.updateTensorboardTask(req.params.id).then((taskDetail: TensorboardTaskInfo) => {
this.tensorboardManager.updateTensorboardTask(req.params['id']).then((taskDetail: TensorboardTaskInfo) => {
res.send(Object.assign({}, taskDetail));
}).catch((err: Error) => {
this.handleError(err, res);
@ -389,7 +387,7 @@ class NNIRestHandler {
private stopTensorboardTask(router: Router): void {
router.delete('/tensorboard/:id', (req: Request, res: Response) => {
this.tensorboardManager.stopTensorboardTask(req.params.id).then((taskDetail: TensorboardTaskInfo) => {
this.tensorboardManager.stopTensorboardTask(req.params['id']).then((taskDetail: TensorboardTaskInfo) => {
res.send(Object.assign({}, taskDetail));
}).catch((err: Error) => {
this.handleError(err, res);
@ -398,7 +396,7 @@ class NNIRestHandler {
}
private stopAllTensorboardTask(router: Router): void {
router.delete('/tensorboard-tasks', (req: Request, res: Response) => {
router.delete('/tensorboard-tasks', (_req: Request, res: Response) => {
this.tensorboardManager.stopAllTensorboardTask().then(() => {
res.send();
}).catch((err: Error) => {
@ -408,7 +406,7 @@ class NNIRestHandler {
}
private listTensorboardTask(router: Router): void {
router.get('/tensorboard-tasks', (req: Request, res: Response) => {
router.get('/tensorboard-tasks', (_req: Request, res: Response) => {
this.tensorboardManager.listTensorboardTasks().then((taskDetails: TensorboardTaskInfo[]) => {
res.send(taskDetails);
}).catch((err: Error) => {
@ -418,7 +416,7 @@ class NNIRestHandler {
}
private stop(router: Router): void {
router.delete('/experiment', (req: Request, res: Response) => {
router.delete('/experiment', (_req: Request, res: Response) => {
this.nniManager.stopExperimentTopHalf().then(() => {
res.send();
this.nniManager.stopExperimentBottomHalf();

2
ts/nni_manager/rest_server/restValidationSchemas.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
const joi = require('joi');
export namespace ValidationSchemas {

2
ts/nni_manager/test/common/getIpv4Address.test.ts
Просмотреть файл

@ -1,7 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
import * as assert from 'assert';
import assert from 'assert';
import { getIPV4Address } from '../../common/utils';
it('getIpv4Address', async () => {

2
ts/nni_manager/test/core/ipcInterfaceTerminate.test.ts
Просмотреть файл

@ -43,7 +43,7 @@ function startProcess(): void {
}
);
const proc: ChildProcess = getTunerProc(dispatcherCmd, stdio, 'core/test', process.env);
proc.on('error', (error: Error): void => {
proc.on('error', (_error: Error): void => {
procExit = true;
procError = true;
});

8
ts/nni_manager/test/core/nnimanager.test.ts
Просмотреть файл

@ -169,9 +169,9 @@ describe('Unit test for nnimanager', function () {
it('test getTrialJob with invalid id', () => {
//query a not exist id, and the function should throw error, and should not process then() method
return nniManager.getTrialJob('4567').then((jobid) => {
return nniManager.getTrialJob('4567').then((_jobid) => {
assert.fail();
}).catch((error) => {
}).catch((_error) => {
assert.isTrue(true);
})
})
@ -252,9 +252,9 @@ describe('Unit test for nnimanager', function () {
it('test getMetricData with invalid trialJobId', () => {
//query an invalid trialJobId
return nniManager.getMetricData('43210', 'CUSTOM').then((metricData) => {
return nniManager.getMetricData('43210', 'CUSTOM').then((_metricData) => {
assert.fail();
}).catch((error) => {
}).catch((_error) => {
})
})

2
ts/nni_manager/test/mock/datastore.ts
Просмотреть файл

@ -204,7 +204,7 @@ class MockedDataStore implements DataStore {
return Promise.resolve(ret);
}
public getTrialJob(trialJobId: string): Promise<TrialJobInfo> {
public getTrialJob(_trialJobId: string): Promise<TrialJobInfo> {
return Promise.resolve({
trialJobId: '1234',
status: 'SUCCEEDED',

6
ts/nni_manager/test/mock/experimentManager.ts
Просмотреть файл

@ -25,16 +25,16 @@ export class mockedeExperimentManager extends ExperimentManager {
"logDir": null
}
}));
return new Promise<JSON>((resolve, reject) => {
return new Promise<JSON>((resolve, _reject) => {
resolve(expInfo);
});
}
public setExperimentPath(newPath: string): void {
public setExperimentPath(_newPath: string): void {
return
}
public setExperimentInfo(experimentId: string, key: string, value: any): void {
public setExperimentInfo(_experimentId: string, _key: string, _value: any): void {
return
}

26
ts/nni_manager/test/mock/nniManager.ts
Просмотреть файл

@ -27,10 +27,10 @@ export class MockedNNIManager extends Manager {
errors: []
}
}
public updateExperimentProfile(experimentProfile: ExperimentProfile, updateType: ProfileUpdateType): Promise<void> {
public updateExperimentProfile(_experimentProfile: ExperimentProfile, _updateType: ProfileUpdateType): Promise<void> {
return Promise.resolve();
}
public importData(data: string): Promise<void> {
public importData(_data: string): Promise<void> {
return Promise.resolve();
}
public getImportedData(): Promise<string[]> {
@ -53,7 +53,7 @@ export class MockedNNIManager extends Manager {
return deferred.promise;
}
public addCustomizedTrialJob(hyperParams: string): Promise<number> {
public addCustomizedTrialJob(_hyperParams: string): Promise<number> {
return Promise.resolve(99);
}
@ -61,7 +61,7 @@ export class MockedNNIManager extends Manager {
return Promise.resolve();
}
public submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
public submitTrialJob(_form: TrialJobApplicationForm): Promise<TrialJobDetail> {
const deferred: Deferred<TrialJobDetail> = new Deferred<TrialJobDetail>();
const jobDetail: TrialJobDetail = {
id: '1234',
@ -82,23 +82,23 @@ export class MockedNNIManager extends Manager {
return deferred.promise;
}
public cancelTrialJobByUser(trialJobId: string): Promise<void> {
public cancelTrialJobByUser(_trialJobId: string): Promise<void> {
return Promise.resolve();
}
public getClusterMetadata(key: string): Promise<string> {
public getClusterMetadata(_key: string): Promise<string> {
return Promise.resolve('METAVALUE1');
}
public startExperiment(experimentParams: ExperimentConfig): Promise<string> {
public startExperiment(_experimentParams: ExperimentConfig): Promise<string> {
return Promise.resolve('id-1234');
}
public setClusterMetadata(key: string, value: string): Promise<void> {
public setClusterMetadata(_key: string, _value: string): Promise<void> {
return Promise.resolve();
}
public getTrialJob(trialJobId: string): Promise<TrialJobInfo> {
public getTrialJob(_trialJobId: string): Promise<TrialJobInfo> {
const deferred: Deferred<TrialJobInfo> = new Deferred<TrialJobInfo>();
const jobInfo: TrialJobInfo = {
trialJobId: '1234',
@ -120,16 +120,16 @@ export class MockedNNIManager extends Manager {
public stopExperimentBottomHalf(): Promise<void> {
throw new MethodNotImplementedError();
}
public getMetricData(trialJobId: string, metricType: MetricType): Promise<MetricDataRecord[]> {
public getMetricData(_trialJobId: string, _metricType: MetricType): Promise<MetricDataRecord[]> {
throw new MethodNotImplementedError();
}
public getMetricDataByRange(minSeqId: number, maxSeqId: number): Promise<MetricDataRecord[]> {
public getMetricDataByRange(_minSeqId: number, _maxSeqId: number): Promise<MetricDataRecord[]> {
throw new MethodNotImplementedError();
}
public getLatestMetricData(): Promise<MetricDataRecord[]> {
throw new MethodNotImplementedError();
}
public getTrialFile(trialJobId: string, fileName: string): Promise<string> {
public getTrialFile(_trialJobId: string, _fileName: string): Promise<string> {
throw new MethodNotImplementedError();
}
public getExperimentProfile(): Promise<ExperimentProfile> {
@ -161,7 +161,7 @@ export class MockedNNIManager extends Manager {
return Promise.resolve(profile);
}
public listTrialJobs(status?: TrialJobStatus): Promise<TrialJobInfo[]> {
public listTrialJobs(_status?: TrialJobStatus): Promise<TrialJobInfo[]> {
const job1: TrialJobInfo = {
trialJobId: '1234',
status: 'SUCCEEDED',

12
ts/nni_manager/test/mock/trainingService.ts
Просмотреть файл

@ -63,7 +63,7 @@ class MockedTrainingService extends TrainingService {
return deferred.promise;
}
public getTrialFile(trialJobId: string, fileName: string): Promise<string> {
public getTrialFile(_trialJobId: string, _fileName: string): Promise<string> {
throw new MethodNotImplementedError();
}
@ -71,18 +71,18 @@ class MockedTrainingService extends TrainingService {
}
public addTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
public addTrialJobMetricListener(_listener: (_metric: TrialJobMetric) => void): void {
}
public removeTrialJobMetricListener(listener: (metric: TrialJobMetric) => void): void {
public removeTrialJobMetricListener(_listener: (_metric: TrialJobMetric) => void): void {
}
public submitTrialJob(form: TrialJobApplicationForm): Promise<TrialJobDetail> {
public submitTrialJob(_form: TrialJobApplicationForm): Promise<TrialJobDetail> {
const deferred = new Deferred<TrialJobDetail>();
return deferred.promise;
}
public updateTrialJob(trialJobId: string, form: TrialJobApplicationForm): Promise<TrialJobDetail> {
public updateTrialJob(_trialJobId: string, _form: TrialJobApplicationForm): Promise<TrialJobDetail> {
throw new MethodNotImplementedError();
}
@ -90,7 +90,7 @@ class MockedTrainingService extends TrainingService {
return false;
}
public cancelTrialJob(trialJobId: string, isEarlyStopped: boolean = false): Promise<void> {
public cancelTrialJob(trialJobId: string, _isEarlyStopped: boolean = false): Promise<void> {
const deferred = new Deferred<void>();
if(trialJobId === '1234' || trialJobId === '3456'){
deferred.resolve();

2
ts/nni_manager/test/register.js Normal file
Просмотреть файл

@ -0,0 +1,2 @@
require('ts-node/register');
require('app-module-path/cwd');

2
ts/nni_manager/test/rest_server/restserver.test.ts
Просмотреть файл

@ -4,7 +4,7 @@
'use strict';
import { assert, expect } from 'chai';
import * as request from 'request';
import request from 'request';
import { Container } from 'typescript-ioc';
import * as component from '../../common/component';

8
ts/nni_manager/test/training_service/adlTrainingService.test.ts
Просмотреть файл

@ -3,10 +3,10 @@
'use strict';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import * as fs from 'fs';
import * as tmp from 'tmp';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import fs from 'fs';
import tmp from 'tmp';
import * as component from '../../common/component';
import { TrialJobApplicationForm, TrialJobDetail, TrainingService } from '../../common/trainingService';
import { cleanupUnitTest, prepareUnitTest } from '../../common/utils';

2
ts/nni_manager/test/training_service/fileUtility.test.ts
Просмотреть файл

@ -11,7 +11,7 @@ import { execCopydir, tarAdd, validateCodeDir } from '../../training_service/com
const deleteFolderRecursive = (filePath: string) => {
if (fs.existsSync(filePath)) {
fs.readdirSync(filePath).forEach((file, index) => {
fs.readdirSync(filePath).forEach((file, _index) => {
const curPath = path.join(filePath, file);
if (fs.lstatSync(curPath).isDirectory()) { // recurse
deleteFolderRecursive(curPath);

8
ts/nni_manager/test/training_service/kubeflowTrainingService.test.ts
Просмотреть файл

@ -3,10 +3,10 @@
'use strict';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import * as fs from 'fs';
import * as tmp from 'tmp';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import fs from 'fs';
import tmp from 'tmp';
import * as component from '../../common/component';
import { cleanupUnitTest, prepareUnitTest } from '../../common/utils';
import { TrialConfigMetadataKey } from '../../training_service/common/trialConfigMetadataKey';

10
ts/nni_manager/test/training_service/localTrainingService.test.ts
Просмотреть файл

@ -3,11 +3,11 @@
'use strict';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import * as fs from 'fs';
import * as path from 'path';
import * as tmp from 'tmp';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import fs from 'fs';
import path from 'path';
import tmp from 'tmp';
import * as component from '../../common/component';
import { TrialJobApplicationForm, TrialJobDetail} from '../../common/trainingService';
import { cleanupUnitTest, delay, prepareUnitTest, getExperimentRootDir } from '../../common/utils';

4
ts/nni_manager/test/training_service/remote/linuxCommands.test.ts
Просмотреть файл

@ -3,8 +3,8 @@
'use strict';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import * as component from '../../../common/component';
import { cleanupUnitTest, prepareUnitTest } from '../../../common/utils';
import { LinuxCommands } from '../../../training_service/remote_machine/extends/linuxCommands';

8
ts/nni_manager/test/training_service/remote/shellExecutor.test.ts
Просмотреть файл

@ -3,10 +3,10 @@
'use strict';
import * as cpp from 'child-process-promise';
import * as fs from 'fs';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import cpp from 'child-process-promise';
import fs from 'fs';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import { ShellExecutor } from '../../../training_service/remote_machine/shellExecutor';
import { prepareUnitTest, cleanupUnitTest } from '../../../common/utils';

4
ts/nni_manager/test/training_service/remote/windowsCommands.test.ts
Просмотреть файл

@ -3,8 +3,8 @@
'use strict';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import * as component from '../../../common/component';
import { cleanupUnitTest, prepareUnitTest } from '../../../common/utils';
import { WindowsCommands } from '../../../training_service/remote_machine/extends/windowsCommands';

14
ts/nni_manager/test/training_service/remoteMachineTrainingService.test.ts
Просмотреть файл

@ -3,11 +3,11 @@
'use strict';
import * as assert from 'assert';
import * as chai from 'chai';
import * as chaiAsPromised from 'chai-as-promised';
import * as fs from 'fs';
import * as tmp from 'tmp';
import assert from 'assert';
import chai from 'chai';
import chaiAsPromised from 'chai-as-promised';
import fs from 'fs';
import tmp from 'tmp';
import * as component from '../../common/component';
import { TrialJobApplicationForm, TrialJobDetail, TrainingService } from '../../common/trainingService';
import { cleanupUnitTest, delay, prepareUnitTest } from '../../common/utils';
@ -133,10 +133,10 @@ describe('Unit Test for RemoteMachineTrainingService', () => {
};
const jobDetail: TrialJobDetail = await remoteMachineTrainingService.submitTrialJob(form);
// Add metrics listeners
const listener1 = function f1(metric: any) {
const listener1 = function f1(_metric: any) {
}
const listener2 = function f1(metric: any) {
const listener2 = function f1(_metric: any) {
}
remoteMachineTrainingService.addTrialJobMetricListener(listener1);

2
ts/nni_manager/test/training_service/reusable/utCommandChannel.ts
Просмотреть файл

@ -26,7 +26,7 @@ export class UtCommandChannel extends CommandChannel {
return this.receivedCommands.shift();
}
public async config(_key: string, value: any): Promise<void> {
public async config(_key: string, _value: any): Promise<void> {
// do nothing
}

2
ts/nni_manager/test/training_service/reusable/utEnvironmentService.ts
Просмотреть файл

@ -58,7 +58,7 @@ export class UtEnvironmentService extends EnvironmentService {
// do nothing
}
public async refreshEnvironmentsStatus(environments: EnvironmentInformation[]): Promise<void> {
public async refreshEnvironmentsStatus(_environments: EnvironmentInformation[]): Promise<void> {
// do nothing
}

24
ts/nni_manager/training_service/common/clusterJobRestServer.ts
Просмотреть файл

@ -1,19 +1,17 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import * as bodyParser from 'body-parser';
import assert from 'assert';
import bodyParser from 'body-parser';
import { Request, Response, Router } from 'express';
import * as fs from 'fs';
import * as path from 'path';
import fs from 'fs';
import path from 'path';
import { Writable } from 'stream';
import { String } from 'typescript-string-operations';
import * as component from '../../common/component';
import { getBasePort, getExperimentId } from '../../common/experimentStartupInfo';
import { RestServer } from '../../common/restServer';
import { getExperimentRootDir, mkDirPSync } from '../../common/utils';
import * as component from 'common/component';
import { getBasePort, getExperimentId } from 'common/experimentStartupInfo';
import { RestServer } from 'common/restServer';
import { getExperimentRootDir, mkDirPSync } from 'common/utils';
/**
* Cluster Job Training service Rest server, provides rest API to support Cluster job metrics update
@ -108,7 +106,7 @@ export abstract class ClusterJobRestServer extends RestServer {
router.post(`/update-metrics/${this.expId}/:trialId`, (req: Request, res: Response) => {
try {
this.log.info(`Get update-metrics request, trial job id is ${req.params.trialId}`);
this.log.info(`Get update-metrics request, trial job id is ${req.params['trialId']}`);
this.log.info('update-metrics body is', req.body);
this.handleTrialMetrics(req.body.jobId, req.body.metrics);
@ -127,7 +125,7 @@ export abstract class ClusterJobRestServer extends RestServer {
this.errorMessage = `Version check failed, didn't get version check response from trialKeeper,`
+ ` please check your NNI version in NNIManager and TrialKeeper!`;
}
const trialLogDir: string = path.join(getExperimentRootDir(), 'trials', req.params.trialId);
const trialLogDir: string = path.join(getExperimentRootDir(), 'trials', req.params['trialId']);
mkDirPSync(trialLogDir);
const trialLogPath: string = path.join(trialLogDir, 'stdout_log_collection.log');
try {
@ -136,7 +134,7 @@ export abstract class ClusterJobRestServer extends RestServer {
const metricsContent: any = req.body.msg.match(this.NNI_METRICS_PATTERN);
if (metricsContent && metricsContent.groups) {
const key: string = 'metrics';
this.handleTrialMetrics(req.params.trialId, [metricsContent.groups[key]]);
this.handleTrialMetrics(req.params['trialId'], [metricsContent.groups[key]]);
skipLogging = true;
}
}

4
ts/nni_manager/training_service/common/containerJobData.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
export const CONTAINER_INSTALL_NNI_SHELL_FORMAT: string =
`#!/bin/bash
if python3 -c 'import nni' > /dev/null 2>&1; then
@ -18,4 +16,4 @@ export const CONTAINER_INSTALL_NNI_SHELL_FORMAT_FOR_WIN: string =
if ($error -ne ''){
python -m pip install --user --upgrade nni
}
exit`;
exit`;

19
ts/nni_manager/training_service/common/gpuData.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
export enum ScheduleResultType {
// Schedule succeeded
SUCCEED,
@ -65,14 +63,15 @@ export class GPUSummary {
export function parseGpuIndices(gpuIndices?: string): Set<number> | undefined {
if (gpuIndices !== undefined) {
const indices: number[] = gpuIndices.split(',')
.map((x: string) => parseInt(x, 10));
if (indices.length > 0) {
return new Set(indices);
} else {
throw new Error('gpuIndices can not be empty if specified.');
}
if (gpuIndices === undefined) {
return undefined;
}
const indices: number[] = gpuIndices.split(',')
.map((x: string) => parseInt(x, 10));
if (indices.length > 0) {
return new Set(indices);
} else {
throw new Error('gpuIndices can not be empty if specified.');
}
}

4
ts/nni_manager/training_service/common/jobMetrics.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { TrialJobStatus } from '../../common/trainingService';
import { TrialJobStatus } from 'common/trainingService';
/**
* Trial job metrics class

2
ts/nni_manager/training_service/common/trialConfig.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
/**
* Trial job configuration class
* Representing trial job configurable properties

2
ts/nni_manager/training_service/common/trialConfigMetadataKey.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
/**
* Enum of metadata keys for configuration
*/

14
ts/nni_manager/training_service/common/util.ts
Просмотреть файл

@ -1,15 +1,13 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as cpp from 'child-process-promise';
import * as cp from 'child_process';
import * as fs from 'fs';
import cpp from 'child-process-promise';
import cp from 'child_process';
import fs from 'fs';
import ignore from 'ignore';
import * as path from 'path';
import * as tar from 'tar';
import { getLogger } from '../../common/log';
import path from 'path';
import tar from 'tar';
import { getLogger } from 'common/log';
import { String } from 'typescript-string-operations';
import { GPU_INFO_COLLECTOR_FORMAT_WINDOWS } from './gpuData';

4
ts/nni_manager/training_service/kubernetes/adl/adlApiClient.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import fs from 'fs';
import { GeneralK8sClient, KubernetesCRDClient } from '../kubernetesApiClient';
/**

2
ts/nni_manager/training_service/kubernetes/adl/adlConfig.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import {KubernetesTrialConfig} from "../kubernetesConfig";
/**

2
ts/nni_manager/training_service/kubernetes/adl/adlJobInfoCollector.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { AdlClientV1 } from './adlApiClient';
import { KubernetesTrialJobDetail} from '../kubernetesData';
import { KubernetesJobInfoCollector } from '../kubernetesJobInfoCollector';

4
ts/nni_manager/training_service/kubernetes/adl/adlJobRestServer.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as component from '../../../common/component';
import * as component from 'common/component';
import { KubernetesJobRestServer } from '../kubernetesJobRestServer';
import { AdlTrainingService } from './adlTrainingService';

14
ts/nni_manager/training_service/kubernetes/adl/adlTrainingService.ts
Просмотреть файл

@ -1,18 +1,16 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import * as component from '../../../common/component';
import fs from 'fs';
import * as component from 'common/component';
import { String } from 'typescript-string-operations';
import { getExperimentId } from '../../../common/experimentStartupInfo';
import { getExperimentId } from 'common/experimentStartupInfo';
import {
NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
} from '../../../common/trainingService';
import { delay, generateParamFileName, getVersion, uniqueString } from '../../../common/utils';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
} from 'common/trainingService';
import { delay, generateParamFileName, getVersion, uniqueString } from 'common/utils';
import { TrialConfigMetadataKey } from 'training_service/common/trialConfigMetadataKey';
import { KubernetesTrialJobDetail } from '../kubernetesData';
import { KubernetesTrainingService } from '../kubernetesTrainingService';
import { AdlClientFactory } from './adlApiClient'

12
ts/nni_manager/training_service/kubernetes/azureStorageClientUtils.ts
Просмотреть файл

@ -1,15 +1,13 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as azureStorage from 'azure-storage';
import * as fs from 'fs';
import * as path from 'path';
import azureStorage from 'azure-storage';
import fs from 'fs';
import path from 'path';
import { Deferred } from 'ts-deferred';
import { String } from 'typescript-string-operations';
import { getLogger } from '../../common/log';
import { mkDirP } from '../../common/utils';
import { getLogger } from 'common/log';
import { mkDirP } from 'common/utils';
export namespace AzureStorageClientUtility {

4
ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerApiClient.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import fs from 'fs';
import {GeneralK8sClient, KubernetesCRDClient} from '../kubernetesApiClient';
/**

6
ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerConfig.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import {
AzureStorage, KeyVaultConfig, KubernetesClusterConfig, KubernetesClusterConfigAzure, KubernetesClusterConfigNFS,
@ -49,7 +47,7 @@ export class FrameworkControllerTrialConfig extends KubernetesTrialConfig {
export class FrameworkControllerClusterConfig extends KubernetesClusterConfig {
public readonly serviceAccountName: string;
constructor(apiVersion: string, serviceAccountName: string, configPath?: string, namespace?: string) {
constructor(apiVersion: string, serviceAccountName: string, _configPath?: string, namespace?: string) {
super(apiVersion, undefined, namespace);
this.serviceAccountName = serviceAccountName;
}

2
ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobInfoCollector.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { KubernetesCRDClient } from '../kubernetesApiClient';
import { KubernetesTrialJobDetail} from '../kubernetesData';
import { KubernetesJobInfoCollector } from '../kubernetesJobInfoCollector';

4
ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerJobRestServer.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as component from '../../../common/component';
import * as component from 'common/component';
import { KubernetesJobRestServer } from '../kubernetesJobRestServer';
import { FrameworkControllerTrainingService } from './frameworkcontrollerTrainingService';

24
ts/nni_manager/training_service/kubernetes/frameworkcontroller/frameworkcontrollerTrainingService.ts
Просмотреть файл

@ -1,21 +1,19 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import * as cpp from 'child-process-promise';
import * as fs from 'fs';
import * as path from 'path';
import * as component from '../../../common/component';
import {getExperimentId} from '../../../common/experimentStartupInfo';
import assert from 'assert';
import cpp from 'child-process-promise';
import fs from 'fs';
import path from 'path';
import * as component from 'common/component';
import {getExperimentId} from 'common/experimentStartupInfo';
import {
NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
} from '../../../common/trainingService';
import {delay, generateParamFileName, getExperimentRootDir, uniqueString} from '../../../common/utils';
import {CONTAINER_INSTALL_NNI_SHELL_FORMAT} from '../../common/containerJobData';
import {TrialConfigMetadataKey} from '../../common/trialConfigMetadataKey';
import {validateCodeDir} from '../../common/util';
} from 'common/trainingService';
import {delay, generateParamFileName, getExperimentRootDir, uniqueString} from 'common/utils';
import {CONTAINER_INSTALL_NNI_SHELL_FORMAT} from 'training_service/common/containerJobData';
import {TrialConfigMetadataKey} from 'training_service/common/trialConfigMetadataKey';
import {validateCodeDir} from 'training_service/common/util';
import {NFSConfig} from '../kubernetesConfig';
import {KubernetesTrialJobDetail} from '../kubernetesData';
import {KubernetesTrainingService} from '../kubernetesTrainingService';

4
ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowApiClient.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import fs from 'fs';
import { GeneralK8sClient, KubernetesCRDClient } from '../kubernetesApiClient';
import { KubeflowOperator } from './kubeflowConfig';

6
ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowConfig.ts
Просмотреть файл

@ -1,10 +1,8 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import { MethodNotImplementedError } from '../../../common/errors';
import assert from 'assert';
import { MethodNotImplementedError } from 'common/errors';
import { AzureStorage, KeyVaultConfig, KubernetesClusterConfig, KubernetesClusterConfigAzure, KubernetesClusterConfigNFS,
KubernetesStorageKind, KubernetesTrialConfig, KubernetesTrialConfigTemplate, NFSConfig, StorageConfig
} from '../kubernetesConfig';

2
ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobInfoCollector.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { KubernetesCRDClient } from '../kubernetesApiClient';
import { KubernetesTrialJobDetail} from '../kubernetesData';
import { KubernetesJobInfoCollector } from '../kubernetesJobInfoCollector';

4
ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowJobRestServer.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as component from '../../../common/component';
import * as component from 'common/component';
import { KubernetesJobRestServer } from '../kubernetesJobRestServer';
import { KubeflowTrainingService } from './kubeflowTrainingService';

24
ts/nni_manager/training_service/kubernetes/kubeflow/kubeflowTrainingService.ts
Просмотреть файл

@ -1,22 +1,20 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import assert from 'assert';
import cpp from 'child-process-promise';
import fs from 'fs';
import path from 'path';
import * as component from 'common/component';
import * as assert from 'assert';
import * as cpp from 'child-process-promise';
import * as fs from 'fs';
import * as path from 'path';
import * as component from '../../../common/component';
import { getExperimentId } from '../../../common/experimentStartupInfo';
import { getExperimentId } from 'common/experimentStartupInfo';
import {
NNIManagerIpConfig, TrialJobApplicationForm, TrialJobDetail, TrialJobStatus
} from '../../../common/trainingService';
import { delay, generateParamFileName, getExperimentRootDir, uniqueString } from '../../../common/utils';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../../common/containerJobData';
import { TrialConfigMetadataKey } from '../../common/trialConfigMetadataKey';
import { validateCodeDir } from '../../common/util';
} from 'common/trainingService';
import { delay, generateParamFileName, getExperimentRootDir, uniqueString } from 'common/utils';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from 'training_service/common/containerJobData';
import { TrialConfigMetadataKey } from 'training_service/common/trialConfigMetadataKey';
import { validateCodeDir } from 'training_service/common/util';
import { NFSConfig } from '../kubernetesConfig';
import { KubernetesTrialJobDetail } from '../kubernetesData';
import { KubernetesJobRestServer } from '../kubernetesJobRestServer';

4
ts/nni_manager/training_service/kubernetes/kubernetesApiClient.ts
Просмотреть файл

@ -1,11 +1,9 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
// eslint-disable-next-line @typescript-eslint/camelcase
import {Client1_10, config} from 'kubernetes-client';
import {getLogger, Logger} from '../../common/log';
import {getLogger, Logger} from 'common/log';
/**
* This function uses the environment variable

4
ts/nni_manager/training_service/kubernetes/kubernetesConfig.ts
Просмотреть файл

@ -1,10 +1,8 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
export type KubernetesStorageKind = 'nfs' | 'azureStorage' | 'pvc';
import {MethodNotImplementedError} from '../../common/errors';
import {MethodNotImplementedError} from 'common/errors';
export abstract class KubernetesClusterConfig {
public readonly storage?: KubernetesStorageKind;

4
ts/nni_manager/training_service/kubernetes/kubernetesData.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from 'common/trainingService';
/**
* KubeflowTrialJobDetail

10
ts/nni_manager/training_service/kubernetes/kubernetesJobInfoCollector.ts
Просмотреть файл

@ -1,12 +1,10 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import { MethodNotImplementedError, NNIError, NNIErrorNames } from '../../common/errors';
import { getLogger, Logger } from '../../common/log';
import { TrialJobStatus } from '../../common/trainingService';
import assert from 'assert';
import { MethodNotImplementedError, NNIError, NNIErrorNames } from 'common/errors';
import { getLogger, Logger } from 'common/log';
import { TrialJobStatus } from 'common/trainingService';
import { KubernetesCRDClient } from './kubernetesApiClient';
import { KubernetesTrialJobDetail } from './kubernetesData';

4
ts/nni_manager/training_service/kubernetes/kubernetesJobRestServer.ts
Просмотреть файл

@ -1,10 +1,8 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { Inject } from 'typescript-ioc';
import * as component from '../../common/component';
import * as component from 'common/component';
import { ClusterJobRestServer } from '../common/clusterJobRestServer';
import { KubernetesTrainingService } from './kubernetesTrainingService';

18
ts/nni_manager/training_service/kubernetes/kubernetesTrainingService.ts
Просмотреть файл

@ -1,22 +1,20 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import cpp from 'child-process-promise';
import path from 'path';
import * as cpp from 'child-process-promise';
import * as path from 'path';
import * as azureStorage from 'azure-storage';
import azureStorage from 'azure-storage';
import {EventEmitter} from 'events';
import {Base64} from 'js-base64';
import {String} from 'typescript-string-operations';
import {getExperimentId} from '../../common/experimentStartupInfo';
import {getLogger, Logger} from '../../common/log';
import {MethodNotImplementedError} from '../../common/errors';
import {getExperimentId} from 'common/experimentStartupInfo';
import {getLogger, Logger} from 'common/log';
import {MethodNotImplementedError} from 'common/errors';
import {
NNIManagerIpConfig, TrialJobDetail, TrialJobMetric
} from '../../common/trainingService';
import {delay, getExperimentRootDir, getIPV4Address, getJobCancelStatus, getVersion, uniqueString} from '../../common/utils';
} from 'common/trainingService';
import {delay, getExperimentRootDir, getIPV4Address, getJobCancelStatus, getVersion, uniqueString} from 'common/utils';
import {AzureStorageClientUtility} from './azureStorageClientUtils';
import {GeneralK8sClient, KubernetesCRDClient} from './kubernetesApiClient';
import {KubernetesClusterConfig} from './kubernetesConfig';

14
ts/nni_manager/training_service/local/gpuScheduler.ts
Просмотреть файл

@ -1,14 +1,12 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as cpp from 'child-process-promise';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import { getLogger, Logger } from '../../common/log';
import { delay } from '../../common/utils';
import cpp from 'child-process-promise';
import fs from 'fs';
import os from 'os';
import path from 'path';
import { getLogger, Logger } from 'common/log';
import { delay } from 'common/utils';
import { GPUInfo, GPUSummary } from '../common/gpuData';
import { execKill, execMkdir, execRemove, execTail, runGpuMetricsCollector } from '../common/util';

27
ts/nni_manager/training_service/local/localTrainingService.ts
Просмотреть файл

@ -1,25 +1,24 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as cp from 'child_process';
import cp from 'child_process';
import { EventEmitter } from 'events';
import * as fs from 'fs';
import * as path from 'path';
import * as ts from 'tail-stream';
import * as tkill from 'tree-kill';
import { NNIError, NNIErrorNames } from '../../common/errors';
import { getExperimentId } from '../../common/experimentStartupInfo';
import { getLogger, Logger } from '../../common/log';
import { powershellString } from '../../common/shellUtils';
import fs from 'fs';
import path from 'path';
import ts from 'tail-stream';
import tkill from 'tree-kill';
import { NNIError, NNIErrorNames } from 'common/errors';
import { getExperimentId } from 'common/experimentStartupInfo';
import { getLogger, Logger } from 'common/log';
import { powershellString } from 'common/shellUtils';
import {
HyperParameters, TrainingService, TrialJobApplicationForm,
TrialJobDetail, TrialJobMetric, TrialJobStatus
} from '../../common/trainingService';
} from 'common/trainingService';
import {
delay, generateParamFileName, getExperimentRootDir, getJobCancelStatus, getNewLine, isAlive, uniqueString
} from '../../common/utils';
import { ExperimentConfig, LocalConfig, flattenConfig } from '../../common/experimentConfig';
} from 'common/utils';
import { ExperimentConfig, LocalConfig, flattenConfig } from 'common/experimentConfig';
import { execMkdir, execNewFile, getScriptName, runScript, setEnvironmentVariable } from '../common/util';
import { GPUScheduler } from './gpuScheduler';
@ -447,7 +446,7 @@ class LocalTrainingService implements TrainingService {
if (process.platform !== 'win32') {
runScriptContent.push('#!/bin/bash');
} else {
runScriptContent.push(`$env:PATH=${powershellString(process.env.path!)}`)
runScriptContent.push(`$env:PATH=${powershellString(process.env['path']!)}`)
}
for (const variable of variables) {
runScriptContent.push(setEnvironmentVariable(variable));

4
ts/nni_manager/training_service/pai/paiConfig.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from 'common/trainingService';
import {TrialConfig} from '../common/trialConfig';
export class PAIClusterConfig {

14
ts/nni_manager/training_service/pai/paiJobInfoCollector.ts
Просмотреть файл

@ -1,14 +1,12 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as request from 'request';
import request from 'request';
import { Deferred } from 'ts-deferred';
import { NNIError, NNIErrorNames } from '../../common/errors';
import { getLogger, Logger } from '../../common/log';
import { TrialJobStatus } from '../../common/trainingService';
import { ExperimentConfig, OpenpaiConfig } from '../../common/experimentConfig';
import { NNIError, NNIErrorNames } from 'common/errors';
import { getLogger, Logger } from 'common/log';
import { TrialJobStatus } from 'common/trainingService';
import { ExperimentConfig, OpenpaiConfig } from 'common/experimentConfig';
import { PAITrialJobDetail } from './paiConfig';
interface FlattenOpenpaiConfig extends ExperimentConfig, OpenpaiConfig { }
@ -44,7 +42,7 @@ export class PAIJobInfoCollector {
await Promise.all(updatePaiTrialJobs);
}
private getSinglePAITrialJobInfo(protocol: string, paiTrialJob: PAITrialJobDetail, paiToken: string, config: FlattenOpenpaiConfig): Promise<void> {
private getSinglePAITrialJobInfo(_protocol: string, paiTrialJob: PAITrialJobDetail, paiToken: string, config: FlattenOpenpaiConfig): Promise<void> {
const deferred: Deferred<void> = new Deferred<void>();
if (!this.statusesNeedToCheck.includes(paiTrialJob.status)) {
deferred.resolve();

4
ts/nni_manager/training_service/pai/paiJobRestServer.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { Request, Response, Router } from 'express';
import { ClusterJobRestServer } from '../common/clusterJobRestServer';
import { PAITrainingService } from './paiTrainingService';
@ -56,7 +54,7 @@ export class PAIJobRestServer extends ClusterJobRestServer {
}
});
router.get(`/parameter-file-meta`, (req: Request, res: Response) => {
router.get(`/parameter-file-meta`, (_req: Request, res: Response) => {
try {
this.log.info(`GET /parameter-file-meta`);
res.send(this.parameterFileMetaList);

24
ts/nni_manager/training_service/pai/paiTrainingService.ts
Просмотреть файл

@ -1,29 +1,27 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as fs from 'fs';
import * as path from 'path';
import * as request from 'request';
import * as component from '../../common/component';
import fs from 'fs';
import path from 'path';
import request from 'request';
import * as component from 'common/component';
import { EventEmitter } from 'events';
import { Deferred } from 'ts-deferred';
import { getExperimentId } from '../../common/experimentStartupInfo';
import { getLogger, Logger } from '../../common/log';
import { MethodNotImplementedError } from '../../common/errors';
import { getExperimentId } from 'common/experimentStartupInfo';
import { getLogger, Logger } from 'common/log';
import { MethodNotImplementedError } from 'common/errors';
import {
HyperParameters, NNIManagerIpConfig, TrainingService,
TrialJobApplicationForm, TrialJobDetail, TrialJobMetric
} from '../../common/trainingService';
import { delay } from '../../common/utils';
import { ExperimentConfig, OpenpaiConfig, flattenConfig, toMegaBytes } from '../../common/experimentConfig';
} from 'common/trainingService';
import { delay } from 'common/utils';
import { ExperimentConfig, OpenpaiConfig, flattenConfig, toMegaBytes } from 'common/experimentConfig';
import { PAIJobInfoCollector } from './paiJobInfoCollector';
import { PAIJobRestServer } from './paiJobRestServer';
import { PAITrialJobDetail, PAI_TRIAL_COMMAND_FORMAT } from './paiConfig';
import { String } from 'typescript-string-operations';
import { generateParamFileName, getIPV4Address, uniqueString } from '../../common/utils';
import { generateParamFileName, getIPV4Address, uniqueString } from 'common/utils';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
import { execMkdir, validateCodeDir, execCopydir } from '../common/util';

2
ts/nni_manager/training_service/remote_machine/extends/linuxCommands.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { OsCommands } from "../osCommands";
import { RemoteCommandResult } from "../remoteMachineData";

2
ts/nni_manager/training_service/remote_machine/extends/windowsCommands.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { OsCommands } from "../osCommands";
import { RemoteCommandResult } from "../remoteMachineData";

11
ts/nni_manager/training_service/remote_machine/gpuScheduler.ts
Просмотреть файл

@ -1,12 +1,10 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import { getLogger, Logger } from '../../common/log';
import { randomSelect } from '../../common/utils';
import { RemoteMachineConfig } from '../../common/experimentConfig';
import assert from 'assert';
import { getLogger, Logger } from 'common/log';
import { randomSelect } from 'common/utils';
import { RemoteMachineConfig } from 'common/experimentConfig';
import { GPUInfo, ScheduleResultType } from '../common/gpuData';
import { ExecutorManager, RemoteMachineMeta, RemoteMachineScheduleResult, RemoteMachineTrialJobDetail } from './remoteMachineData';
@ -123,6 +121,7 @@ export class GPUScheduler {
assert(false, 'gpuInfos is undefined');
}
}
return undefined;
}
/**

2
ts/nni_manager/training_service/remote_machine/osCommands.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { RemoteCommandResult } from "./remoteMachineData";
abstract class OsCommands {

6
ts/nni_manager/training_service/remote_machine/remoteMachineData.ts
Просмотреть файл

@ -1,10 +1,8 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from '../../common/trainingService';
import { RemoteMachineConfig } from '../../common/experimentConfig';
import { TrialJobApplicationForm, TrialJobDetail, TrialJobStatus } from 'common/trainingService';
import { RemoteMachineConfig } from 'common/experimentConfig';
import { GPUInfo, GPUSummary, ScheduleResultType } from '../common/gpuData';
import { ShellExecutor } from './shellExecutor';

2
ts/nni_manager/training_service/remote_machine/remoteMachineJobRestServer.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { ClusterJobRestServer } from '../common/clusterJobRestServer';
import { RemoteMachineTrainingService } from './remoteMachineTrainingService';

24
ts/nni_manager/training_service/remote_machine/remoteMachineTrainingService.ts
Просмотреть файл

@ -1,28 +1,26 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import assert from 'assert';
import { EventEmitter } from 'events';
import * as fs from 'fs';
import * as path from 'path';
import fs from 'fs';
import path from 'path';
import { ShellExecutor } from 'training_service/remote_machine/shellExecutor';
import { Deferred } from 'ts-deferred';
import * as component from '../../common/component';
import { NNIError, NNIErrorNames, MethodNotImplementedError } from '../../common/errors';
import { getExperimentId } from '../../common/experimentStartupInfo';
import { getLogger, Logger } from '../../common/log';
import { ObservableTimer } from '../../common/observableTimer';
import * as component from 'common/component';
import { NNIError, NNIErrorNames, MethodNotImplementedError } from 'common/errors';
import { getExperimentId } from 'common/experimentStartupInfo';
import { getLogger, Logger } from 'common/log';
import { ObservableTimer } from 'common/observableTimer';
import {
HyperParameters, TrainingService, TrialJobApplicationForm,
TrialJobDetail, TrialJobMetric
} from '../../common/trainingService';
} from 'common/trainingService';
import {
delay, generateParamFileName, getExperimentRootDir, getIPV4Address, getJobCancelStatus,
getVersion, uniqueString
} from '../../common/utils';
import { ExperimentConfig, RemoteConfig, RemoteMachineConfig, flattenConfig } from '../../common/experimentConfig';
} from 'common/utils';
import { ExperimentConfig, RemoteConfig, RemoteMachineConfig, flattenConfig } from 'common/experimentConfig';
import { CONTAINER_INSTALL_NNI_SHELL_FORMAT } from '../common/containerJobData';
import { GPUSummary, ScheduleResultType } from '../common/gpuData';
import { execMkdir, validateCodeDir } from '../common/util';

18
ts/nni_manager/training_service/remote_machine/shellExecutor.ts
Просмотреть файл

@ -1,23 +1,21 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as assert from 'assert';
import * as fs from 'fs';
import * as os from 'os';
import * as path from 'path';
import assert from 'assert';
import fs from 'fs';
import os from 'os';
import path from 'path';
import { Client, ClientChannel, ConnectConfig, SFTPWrapper } from 'ssh2';
import * as stream from 'stream';
import stream from 'stream';
import { Deferred } from "ts-deferred";
import { getLogger, Logger } from '../../common/log';
import { uniqueString, randomInt } from '../../common/utils';
import { getLogger, Logger } from 'common/log';
import { uniqueString, randomInt } from 'common/utils';
import { execRemove, tarAdd } from '../common/util';
import { LinuxCommands } from "./extends/linuxCommands";
import { WindowsCommands } from './extends/windowsCommands';
import { OsCommands } from "./osCommands";
import { RemoteCommandResult, RemoteMachineMeta } from "./remoteMachineData";
import { NNIError, NNIErrorNames } from '../../common/errors';
import { NNIError, NNIErrorNames } from 'common/errors';
class ShellExecutor {
public name: string = "";

2
ts/nni_manager/training_service/reusable/aml/amlClient.ts
Просмотреть файл

@ -1,8 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { Deferred } from 'ts-deferred';
import { PythonShell } from 'python-shell';

4
ts/nni_manager/training_service/reusable/aml/amlConfig.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { TrialConfig } from '../../common/trialConfig';
import { TrialConfig } from 'training_service/common/trialConfig';
import { EnvironmentInformation } from '../environment';
import { AMLClient } from '../aml/amlClient';

4
ts/nni_manager/training_service/reusable/channels/amlCommandChannel.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { delay } from "../../../common/utils";
import { delay } from "common/utils";
import { AMLEnvironmentInformation } from '../aml/amlConfig';
import { CommandChannel, RunnerConnection } from "../commandChannel";
import { Channel, EnvironmentInformation } from "../environment";

6
ts/nni_manager/training_service/reusable/channels/fileCommandChannel.ts
Просмотреть файл

@ -1,10 +1,8 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import * as component from "../../../common/component";
import { delay } from "../../../common/utils";
import * as component from "common/component";
import { delay } from "common/utils";
import { CommandChannel, RunnerConnection } from "../commandChannel";
import { Channel, EnvironmentInformation } from "../environment";
import { StorageService } from "../storageService";

6
ts/nni_manager/training_service/reusable/channels/webCommandChannel.ts
Просмотреть файл

@ -1,11 +1,9 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { Server as SocketServer } from "ws";
import { getBasePort, getExperimentId } from "../../../common/experimentStartupInfo";
import { INITIALIZED } from '../../../core/commands';
import { getBasePort, getExperimentId } from "common/experimentStartupInfo";
import { INITIALIZED } from 'core/commands';
import { CommandChannel, RunnerConnection } from "../commandChannel";
import { Channel, EnvironmentInformation } from "../environment";
import { EventEmitter } from "events";

8
ts/nni_manager/training_service/reusable/commandChannel.ts
Просмотреть файл

@ -1,12 +1,10 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { EventEmitter } from "events";
import { getLogger, Logger } from "../../common/log";
import { TRIAL_COMMANDS } from "../../core/commands";
import { encodeCommand } from "../../core/ipcInterface";
import { getLogger, Logger } from "common/log";
import { TRIAL_COMMANDS } from "core/commands";
import { encodeCommand } from "core/ipcInterface";
import { Channel, EnvironmentInformation } from "./environment";
const acceptedCommands: Set<string> = new Set<string>(TRIAL_COMMANDS);

4
ts/nni_manager/training_service/reusable/dlc/dlcClient.ts
Просмотреть файл

@ -1,11 +1,9 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { Deferred } from 'ts-deferred';
import { PythonShell } from 'python-shell';
import { getLogger, Logger } from '../../../common/log';
import { getLogger, Logger } from 'common/log';
export class DlcClient {
private log: Logger;

4
ts/nni_manager/training_service/reusable/dlc/dlcConfig.ts
Просмотреть файл

@ -1,9 +1,7 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.
'use strict';
import { TrialConfig } from '../../common/trialConfig';
import { TrialConfig } from 'training_service/common/trialConfig';
import { EnvironmentInformation } from '../environment';
import { DlcClient } from '../dlc/dlcClient';

Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше