зеркало из https://github.com/microsoft/nni.git
fix no log from subprocess on trial (#3653)
Co-authored-by: Hao Ni <v-nihao@microsoft.com>
This commit is contained in:
Родитель
277e63f27a
Коммит
684005d7dc
|
@ -8,7 +8,7 @@
|
|||
*/
|
||||
type TrialJobStatus = 'UNKNOWN' | 'WAITING' | 'RUNNING' | 'SUCCEEDED' | 'FAILED' | 'USER_CANCELED' | 'SYS_CANCELED' | 'EARLY_STOPPED';
|
||||
|
||||
type LogType = 'TRIAL_LOG' | 'TRIAL_ERROR';
|
||||
type LogType = 'TRIAL_LOG' | 'TRIAL_STDOUT' | 'TRIAL_ERROR';
|
||||
|
||||
interface TrainingServiceMetadata {
|
||||
readonly key: string;
|
||||
|
|
|
@ -174,6 +174,8 @@ class LocalTrainingService implements TrainingService {
|
|||
let logPath: string;
|
||||
if (logType === 'TRIAL_LOG') {
|
||||
logPath = path.join(this.rootDir, 'trials', trialJobId, 'trial.log');
|
||||
} else if (logType === 'TRIAL_STDOUT'){
|
||||
logPath = path.join(this.rootDir, 'trials', trialJobId, 'stdout');
|
||||
} else if (logType === 'TRIAL_ERROR') {
|
||||
logPath = path.join(this.rootDir, 'trials', trialJobId, 'stderr');
|
||||
} else {
|
||||
|
@ -412,15 +414,16 @@ class LocalTrainingService implements TrainingService {
|
|||
private getScript(workingDirectory: string): string[] {
|
||||
const script: string[] = [];
|
||||
if (process.platform === 'win32') {
|
||||
script.push(`$PSDefaultParameterValues = @{'Out-File:Encoding' = 'utf8'}`);
|
||||
script.push(`cd $env:NNI_CODE_DIR`);
|
||||
script.push(
|
||||
`cmd.exe /c ${this.config.trialCommand} 2>&1 | Out-File "${path.join(workingDirectory, 'stderr')}" -encoding utf8`,
|
||||
`cmd.exe /c ${this.config.trialCommand} 1>${path.join(workingDirectory, 'stdout')} 2>${path.join(workingDirectory, 'stderr')}`,
|
||||
`$NOW_DATE = [int64](([datetime]::UtcNow)-(get-date "1/1/1970")).TotalSeconds`,
|
||||
`$NOW_DATE = "$NOW_DATE" + (Get-Date -Format fff).ToString()`,
|
||||
`Write $LASTEXITCODE " " $NOW_DATE | Out-File "${path.join(workingDirectory, '.nni', 'state')}" -NoNewline -encoding utf8`);
|
||||
} else {
|
||||
script.push(`cd $NNI_CODE_DIR`);
|
||||
script.push(`eval ${this.config.trialCommand} 2>"${path.join(workingDirectory, 'stderr')}"`);
|
||||
script.push(`eval ${this.config.trialCommand} 1>${path.join(workingDirectory, 'stdout')} 2>${path.join(workingDirectory, 'stderr')}`);
|
||||
if (process.platform === 'darwin') {
|
||||
// https://superuser.com/questions/599072/how-to-get-bash-execution-time-in-milliseconds-under-mac-os-x
|
||||
// Considering the worst case, write 999 to avoid negative duration
|
||||
|
|
Загрузка…
Ссылка в новой задаче