fix no log from subprocess on trial (#3653)

Co-authored-by: Hao Ni <v-nihao@microsoft.com>
This commit is contained in:
Ni Hao 2021-05-27 15:32:33 +08:00 коммит произвёл GitHub
Родитель 277e63f27a
Коммит 684005d7dc
Не найден ключ, соответствующий данной подписи
Идентификатор ключа GPG: 4AEE18F83AFDEB23
2 изменённых файлов: 6 добавлений и 3 удалений

Просмотреть файл

@ -8,7 +8,7 @@
*/
type TrialJobStatus = 'UNKNOWN' | 'WAITING' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'USER_CANCELED' | 'SYS_CANCELED' | 'EARLY_STOPPED';
type LogType = 'TRIAL_LOG' | 'TRIAL_ERROR';
type LogType = 'TRIAL_LOG' | 'TRIAL_STDOUT' | 'TRIAL_ERROR';
interface TrainingServiceMetadata {
readonly key: string;

Просмотреть файл

@ -174,6 +174,8 @@ class LocalTrainingService implements TrainingService {
let logPath: string;
if (logType === 'TRIAL_LOG') {
logPath = path.join(this.rootDir, 'trials', trialJobId, 'trial.log');
} else if (logType === 'TRIAL_STDOUT'){
logPath = path.join(this.rootDir, 'trials', trialJobId, 'stdout');
} else if (logType === 'TRIAL_ERROR') {
logPath = path.join(this.rootDir, 'trials', trialJobId, 'stderr');
} else {
@ -412,15 +414,16 @@ class LocalTrainingService implements TrainingService {
private getScript(workingDirectory: string): string[] {
const script: string[] = [];
if (process.platform === 'win32') {
script.push(`$PSDefaultParameterValues = @{'Out-File:Encoding' = 'utf8'}`);
script.push(`cd $env:NNI_CODE_DIR`);
script.push(
`cmd.exe /c ${this.config.trialCommand} 2>&1 | Out-File "${path.join(workingDirectory, 'stderr')}" -encoding utf8`,
`cmd.exe /c ${this.config.trialCommand} 1>${path.join(workingDirectory, 'stdout')} 2>${path.join(workingDirectory, 'stderr')}`,
`$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`,
`$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`,
`Write $LASTEXITCODE " " $NOW_DATE | Out-File "${path.join(workingDirectory, '.nni', 'state')}" -NoNewline -encoding utf8`);
} else {
script.push(`cd $NNI_CODE_DIR`);
script.push(`eval ${this.config.trialCommand} 2>"${path.join(workingDirectory, 'stderr')}"`);
script.push(`eval ${this.config.trialCommand} 1>${path.join(workingDirectory, 'stdout')} 2>${path.join(workingDirectory, 'stderr')}`);
if (process.platform === 'darwin') {
// https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x
// Considering the worst case, write 999 to avoid negative duration