diff --git a/Scripts/README.md b/Scripts/README.md index 10ba16ee1..394bca63f 100644 --- a/Scripts/README.md +++ b/Scripts/README.md @@ -1,13 +1,19 @@ -This directory contains different scripts to support CNTK. +This directory contains different scripts to support CNTK. ## CNTK Binary Installers The directory `install` contains scripts which are used in the CNTK binary download to install -CNTK on a users system. They are not intended to run from this location in the repository. +CNTK on a users system. They are NOT intended to run from this location in the repository. * `install/windows` - A script for installing a Windows CNTK *binary* drop, cf. [here](https://github.com/Microsoft/CNTK/wiki/Setup-Windows-Binary-Script). * `install/linux` - A script for installing a Linux CNTK *binary* drop, cf. [here](https://github.com/Microsoft/CNTK/wiki/Setup-Linux-Binary-Script). +## CNTK Development Environment Installer + +The directory `devInstall` contains scripts which are used to create setup an environment to build CNTK from source. They are intended to run from this location in the repository. + +* `devInstall/Windows` - Create a Visual Studio based development environment on Windows + ## CNTK Text format Converters Two Python Scripts for converting Data to CNTK Text format for using as an input for CNTK Text Format diff --git a/Scripts/devInstall/Windows/DevInstall.ps1 b/Scripts/devInstall/Windows/DevInstall.ps1 new file mode 100644 index 000000000..ffa06f536 --- /dev/null +++ b/Scripts/devInstall/Windows/DevInstall.ps1 @@ -0,0 +1,188 @@ +<# + .SYNOPSIS + Use this cmdlet to install a CNTK development environment on your machine + + .DESCRIPTION + The script will download and install the files necessary to create a CNTK development environment on your system. + + It will analyse your machine and will determine which components are required. + The required components will be downloaded in [c:\installCacheCntk] + Repeated operation of this script will reuse already downloaded components. + + .PARAMETER Execute + This is an optional parameter. Without setting this switch, no changes to the machine setup/installation will be performed + + .PARAMETER NoGpu + This is an optional parameter. By setting this switch the GPU specific tools (Cuda, CuDnn, Cub) will not be installed. + + .PARAMETER localCache + This optional parameter can be used to specify the directory downloaded components will be stored in + + .PARAMETER ServerLocation + This is an optional parameter. The script can install pre-compiled components, this parameter + specifies the location on a server where this componentents are downloaded from. + This is useful for a team environment to share the components which need to get compiled (Protobuf, Zlib, libzip) + + .PARAMETER CloneDirectory + By default the installer should be executed out of the \Scripts\devInstall\Windows directory. Out of this + location the installer computes the root directory of the CNTK clone. In the case the installer is in a different location, + the root directory of the CNTK clone can be specified using this parameter + +.EXAMPLE + installer.ps1 + + Run the installer and see what operations would be performed +.EXAMPLE + installer.ps1 -Execute + + Run the installer and install the development tools +.EXAMPLE + installer.ps1 -Execute -NoGpu + + Run the installer, but don't install any GPU specific tools +.EXAMPLE + installer.ps1 -Execute -NoGpu -CloneDirectory c:\repos\CNTKAlternate + + Run the installer, but don't install any GPU specific tools +.EXAMPLE + +#> + +[CmdletBinding()] +Param( + [parameter(Mandatory=$false)] [switch] $Execute, + [parameter(Mandatory=$false)] [switch] $NoGpu, + [parameter(Mandatory=$false)] [string] $localCache = "c:\installCacheCntk", + [parameter(Mandatory=$false)] [string] $InstallLocation = "c:\local", + [parameter(Mandatory=$false)] [string] $ServerLocation, + [parameter(Mandatory=$false)] [string] $CloneDirectory) + + + + $Execute = $true + + +$roboCopyCmd = "robocopy.exe" +$localDir = $InstallLocation + + +# Get the current script's directory and Dot-source the a file with common Powershell script function residing in the the current script's directory +$MyDir = Split-Path $MyInvocation.MyCommand.Definition + +if ($CloneDirectory) { + $reponame = Split-Path $CloneDirectory -Leaf + $repositoryRootDir = Split-Path $CloneDirectory + + $solutionfile = join-path $clontDirectory "CNTK.SLN" + + if (-not (Test-Path -Path $solutionFile -PathType Leaf)) { + Write-Warning "[$CloneDirectory] was specified as the location of the CNTK sourcecode directory." + Write-Warning "The specified directory is not a valid clone of the CNTK Github project." + throw "Terminating install operation" + } +} +else { + $CloneDirectory = Split-Path $mydir + $CloneDirectory = Split-Path $CloneDirectory + $CloneDirectory = Split-Path $CloneDirectory + + $reponame = Split-Path $CloneDirectory -Leaf + $repositoryRootDir = Split-Path $CloneDirectory + + $solutionfile = join-path $CloneDirectory "CNTK.SLN" + + if (-not (Test-Path -Path $solutionFile -PathType Leaf)) { + Write-Warning "The install script was started out of the [$mydir] location. Based on this" + Write-Warning "[$CloneDirectory] should be the location of the CNTK sourcecode directory." + Write-Warning "The specified directory is not a valid clone of the CNTK Github project." + throw "Terminating install operation" + } +} + +. "$MyDir\helper\Display" +. "$MyDir\helper\Common" +. "$MyDir\helper\Operations" +. "$MyDir\helper\Verification" +. "$MyDir\helper\Download" +. "$MyDir\helper\Action" +. "$MyDir\helper\PreRequisites" + +Function main +{ + try { if (-not (DisplayStart)) { + Write-Host + Write-Host " ... Quitting ... " + Write-Host + return + } + + if (-not (Test-Path -Path $localCache)) { + new-item -Path $localcache -ItemType Container -ErrorAction Stop | Out-Null + } + + ClearScriptVariables + + $operation = @(); + $operation += OpScanProgram + $operation += OpCheckVS15Update3 + + if (-not $NoGpu) { + $operation += OpCheckCuda8 + + $operation += OpNVidiaCudnn5180 -cache $localCache -targetFolder $localDir + $operation += OpNvidiaCub141 -cache $localCache -targetFolder $localDir + } + + $operation += OpCMake362 -cache $localCache + $operation += OpMSMPI70 -cache $localCache + $operation += OpMSMPI70SDK -cache $localCache + $operation += OpBoost160VS15 -cache $localCache -targetFolder $localDir + $operation += OpCNTKMKL3 -cache $localCache -targetFolder $localDir + $operation += OpSwig3010 -cache $localCache -targetFolder $localDir + $operation += OpProtoBuf310VS15 -cache $localCache -targetFolder $localDir + $operation += OpZlibVS15 -cache $localCache -targetFolder $localDir + if ($ServerLocation) { + $operation += OpProtoBuf310VS15Internal -server $ServerLocation -cache $localCache -targetFolder $localDir + $operation += OpZLibVS15Internal -server $ServerLocation -cache $localCache -targetFolder $localDir + } + $operation += OpAnaconda3411 -cache $localCache -targetFolder $localDir + $operation += OpAnacondaEnv34 -targetFolder $localDir -repoDir $repositoryRootDir -repoName $reponame + + + #$operation += OpGit2101 -cache $localCache + #$operation += OpGitClone -targetFolder $repositoryRootDir -targetDir $reponame + #$operation += OpSysinternals -cache $localCache -targetFolder $localDir + #$operation += OpOpenCVInternal $ServerLocation -cache $localCache -targetFolder $localDir + #$operation += OpOpenCV31 -cache $localCache -targetFolder $localDir + #$operation += OpCygwin -cache $localCache -targetFolder $localDir + + #$operation += AddOpDisableJITDebug + #$operation += OpTestData "c:\Data\CNTKTestData" "\\storage.ccp.philly.selfhost.corp.microsoft.com\public\CNTKTestData" + #$operation += OpSysinternals -cache $localCache -targetFolder $localDir + + + $operationList = @() + $operationList += (VerifyOperations $operation) + + PreReqOperations $operationList + + if (DisplayAfterVerify $operationList) { + + DownloadOperations $operationList + + ActionOperations $operationList + + #DisplayEnd + } + } + catch { + Write-Host "Exception caught - function main / failure" + Write-Host ($Error[0]).Exception + } +} + +main + +exit 0 + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Action.ps1 b/Scripts/devInstall/Windows/helper/Action.ps1 new file mode 100644 index 000000000..d8f8c8bce --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Action.ps1 @@ -0,0 +1,782 @@ +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# +function ActionOperations( + [parameter(Mandatory=$true)][array] $actionList) +{ + Write-Host "Performing install operations" + + foreach ($item in $actionList) { + if ($item.ActionInfo) { + Write-Host $item.ActionInfo + } + foreach ($actionItem in $item.Action) { + ActionItem $actionItem + } + } + + Write-Host "Install operations finished" + Write-Host +} + +function ActionItem( + [hashtable] $item) +{ + $func = $item["Function"] + + $expr = $func +' $item' + + Write-Verbose "Calling Operation: [$func]" + Invoke-Expression $expr +} + +function InstallExeForPlatform( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $platform = $table["Platform"] + + if (PlatformMatching $platform) { + InstallExe $table + } + return +} + +function InstallExe( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $cmd = $table["Command"] + $param= $table["Param"] + $dir = $table["WorkDir"] + $processWait = $table["ProcessWait"] + $message = $table["message"] + $runAs = GetTableDefaultBool -table $table -entryName "runAs" -defaultValue $true + $maxErrorLevel = GetTableDefaultInt -table $table -entryName "maxErrorLevel" -defaultValue 0 + + if ($message -ne $null) { + Write-Host $message + } + + if ($dir -eq $null) { + DoProcess -doExecute $Execute -command $cmd -param $param -requiresRunAs $runAs -maxErrorLevel $maxErrorLevel -throwOnError $true + } + else { + DoProcess -doExecute $Execute -command $cmd -param $param -requiresRunAs $runAs -workingDir $dir -maxErrorLevel $maxErrorLevel -throwOnError $true + } + + if ( ($processWait -ne $null) -and ($Execute) -and ($false) ) { + do { + start-sleep 20 + $pwait = Get-Process $processWait -ErrorAction SilentlyContinue + } while (-not ($pwait -eq $null)) + } +} + +function InstallYml( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $basePath = $table["BasePath"] + $env= $table["Env"] + $ymlFile = $table["ymlFile"] + + $envsDir = join-path $basePath "envs" + $targetDir = join-path $envsDir $env + + if (test-path -path $targetDir -PathType Container) { + $newTable = @{ Function = "InstallExe"; Command = "$basepath\Scripts\conda.exe"; Param = "env update --file $ymlFile --name $targetDir"; WorkDir = "$basePath\Scripts"; runAs=$false } + } + else { + $newTable = @{ Function = "InstallExe"; Command = "$basepath\Scripts\conda.exe"; Param = "env create --file $ymlFile --prefix $targetDir"; WorkDir = "$basePath\Scripts"; runAs=$false } + } + + InstallExe $newTable +} + +function ExecuteApplication( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $appName = $table["AppName"] + $param= $table["Param"] + $dir = $table["WorkDir"] + $appDir = GetTableDefaultString -table $table -entryName "AppDir" -defaultValue [string]::Empty + $usePath = GetTableDefaultBool -table $table -entryName "UseEnvPath" -defaultValue $false + $maxErrorLevel = GetTableDefaultInt -table $table -entryName "maxErrorLevel" -defaultValue 0 + + if (-not $Execute) { + Write-Host "** Running in DEMOMODE - setting Exit Code **: 0" + return + } + $application = ResolveApplicationName $appName $appDir $usePath + if ($application.Length -eq 0) { + throw "ExecuteApplication: Couldn't resolve program [$appName] with location directory [$appDir] and usePath [$usePath]" + } + if ($dir -eq $null) { + DoProcess -doExecute $Execute -command $application -param $param -maxErrorLevel $maxErrorLevel -throwOnError $true + } + else { + DoProcess -doExecute $Execute -command $application -param $param -workingDir $dir -maxErrorLevel $maxErrorLevel -throwOnError $true + } +} + +function InstallWheel( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $BasePath = $table["BasePath"] + $EnvName = $table["EnvName"] + $whl = $table["whl"] + $message = $table["message"] + $whlDirectory = $table["WheelDirectory"] + + Write-Host $message + if (-not $Execute) { + Write-Host "** Running in DEMOMODE - setting Exit Code **: 0" + return + } + $condaExe = Join-Path $BasePath 'Scripts\conda.exe' + $newPaths = Invoke-DosCommand $condaExe (Write-Output ..activate cmd.exe $EnvName) -maxErrorLevel 0 + + $oldPath = $env:PATH + $env:PATH = $newPaths + ';' + $env:PATH + + Invoke-DosCommand pip (Write-Output install $whl) -maxErrorLevel 0 + $env:PATH = $oldPath + return +} + +function InstallMSI( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + + $func = $table["Function"] + $msi = $table["MsiName"] + $dir = $table["MsiDir"] + + $cmd = "c:\Windows\System32\MSIEXEC.EXE" + $param= "/i $dir\$msi /quiet /norestart" + + DoProcess -doExecute $Execute -command $cmd -param "$param" -requiresRunAs $true -maxErrorLevel 0 -throwOnError $true +} + +function MakeDirectory( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $path = $table["Path"] + + if (-not (test-path -path $path)) { + if ($Execute) { + New-Item $path -type directory | Out-Null + } + } + + return +} + +function RobocopyFromLocalCache( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $source = $table["Source"] + $destination = $table["Destination"] + + RobocopySourceDestination $source $destination +} + +function RobocopySourceDestination( + [Parameter(Mandatory = $true)][string] $source, + [Parameter(Mandatory = $true)][string] $destination, + [bool] $copyAdditive=$false) +{ + if (-not (test-path $source -PathType Any)) { + throw SourceDirectory [$source] is missing + } + + + $option = "/NFL /copy:DT /dcopy:D /xj" + if (-not $copyAdditive) { + $option += " /MIR " + } + + $param = "$source $destination $option" + + DoProcess -doExecute $Execute -command $roboCopyCmd -param $param -maxErrorLevel 8 -throwOnError $true + return +} + +function SetEnvironmentVariable( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $name = $table["EnvVar"] + $content = $table["Content"] + $location = "Machine" + + if (-not $Execute) { + return + } + else { + $demoMessage = "" + } + + if ($demoMessage.Length -gt 0) { + Write-Verbose "$demoMessage[$func]: [$name] = [$content] in [$location]" + } + + SetEnvVar -name "$name" -content "$content" + return +} + +function AddToPath( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $dir = $table["Dir"] + $atStart = $table["AtStart"] + $env = $table["env"] + + if ($env.Length -eq 0) { + $env = "PATH" + } + + $pathValue = [environment]::GetEnvironmentVariable($env, "Machine") + if ($pathValue -eq $null) { + $pathValue = "" + } + $pv = $pathValue.ToLower() + $ap = $dir.ToLower() + + if ($pv.Contains("$ap")) { + Write-Verbose "AddToPath - path information already up-to-date" + return + } + + Write-Verbose "Adding [$dir] to environment [$env]" + if ($atStart) { + $pathvalue = "$dir;$pathvalue" + } + else { + $pathvalue = "$pathvalue;$dir" + } + if ($Execute) { + SetEnvVar -name $env -content "$pathvalue" + } + return +} + +function ExtractAllFromZipForPlatform( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $platform = $table["Platform"] + + if (PlatformMatching $platform) { + ExtractAllFromZip $table + } +} + +function ExtractAllFromZip( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $zipFileName = $table["zipFileName"] + $destination = $table["destination"] + $destinationFolder = $table["destinationFolder"] + $zipSubTree = $table["zipSubTree"] + $copyAdditive = GetTableDefaultBool -table $table -entryName "AddToDirectory" -defaultValue $false + + Write-Verbose "ExtractAllFromZip: zipFileName [$zipFileName] destination [$destination] Folder [$destinationFolder]" + + if (-not $Execute) { + return + } + $completeDestination = join-path -Path $destination -ChildPath $destinationFolder + + if (-not (test-path -path $completeDestination)) { + new-item $completeDestination -type directory -Force -ErrorAction Stop | Out-Null + } + if (-not (test-path $zipFileName -PathType Leaf)) { + throw "ExtractAllFromZip: zipFileName [$zipFileName] not found!" + } + + $tempDir = [System.IO.Path]::GetTempFileName(); + + remove-item $tempDir | Out-Null + + $completeTempDestination = join-path -Path $tempDir -ChildPath $destinationFolder + new-item -type directory -path $completeTempDestination -Force -ErrorAction Stop | Out-Null + + + if ($Execute) { + $obj = new-object -com shell.application + $zipFile = $obj.NameSpace($zipFileName) + $destinationNS = $obj.NameSpace($completeTempDestination) + + $destinationNS.CopyHere($zipFile.Items()) + + if ($zipSubTree -ne $null) { + $completeTempDestination = join-path $completeTempDestination $zipSubTree + } + + RobocopySourceDestination $completeTempDestination $completeDestination $copyAdditive + + rm -r tempDir -Force -ErrorAction SilentlyContinue | Out-Null + } + + return +} + +function ExtractAllFromTarGz( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $sourceFile = $table["SourceFile"] + $targzFileName = $table["TargzFileName"] + $destination = $table["destination"] + + Write-Verbose "ExtractAllFromTarGz: targzFileName [$targzFileName] destination [$destination] in TargetFolder [$targetfolder]" + if (-not $Execute) { + return + } + + $appDir = join-path $env:ProgramFiles "git\usr\bin" + $tarApp = "tar.exe" + + if (-not (test-path -path "$appDir\$tarApp" -PathType Leaf)) { + throw "Unpacking the file [$targzFileName] requires extraction utility [$appDir\$tarApp].\n The utility wasn't found" + } + + Copy-Item $sourceFile "$destination\$targzFileName" -ErrorAction SilentlyContinue + + $dosCommand = @" +set path="$appDir";%PATH% & tar.exe -xz --force-local -f "$destination\$targzFileName" -C "$destination" +"@ + + & cmd /c $dosCommand + if ($LASTEXITCODE -gt 0) { + throw "Running [$appDir\tar.exe] Command failed with exit code $LASTEXITCODE" + } + + Remove-Item "$destination\$targzFileName" -ErrorAction SilentlyContinue + + return +} + +function CreateBatch( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $filename = $table["Filename"] + + if (-not $Execute) { + Write-Host "Create-Batch [$filename]:No-Execute flag. No file created" + return + } + + Remove-Item -Path $filename -ErrorAction SilentlyContinue | Out-Null + + $batchScript = @" +@echo off +if /I "%CMDCMDLINE%" neq ""%COMSPEC%" " ( + echo. + echo Please execute this script from inside a regular Windows command prompt. + echo. + exit /b 0 +) +set PATH=$cntkRootDir\cntk;%PATH% +"$AnacondaBasePath\Scripts\activate" "$AnacondaBasePath\envs\cntk-py34" +"@ + + add-content -Path $filename -Encoding Ascii -Value $batchScript +} + +function SetRegistryKey( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $key = $table["key"] + $elevated = $table["Elevated"] + + if ($Execute) { + $result = Test-Path -Path $key + + if (-not $result) { + Write-Verbose "[$func]: [$key] will be created" + if ($elevated) { + $commandString = "& { new-item -Path '$key' }" + RunPowershellCommand -command "$commandString" -elevated $true -maxErrorLevel 0 + } + else { + new-item -Path $key + } + } + } + return +} + +function SetRegistryKeyNameData( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $key = $table["key"] + $regName = $table["RegName"] + $data = $table["data"] + $dataType = $table["dataType"] + $elevated = GetTableDefaultBool -table $table -entryName "Elevated" -defaultValue $true + + if ($Execute) { + $tab = @{Function = "SetRegistryKey"; Key=$key; Elevated=$elevated} + SetRegistryKey $tab + + Write-Verbose "[$func]: [$key]:[$regName] will be set to [$dataType]:[$data]" + + $commandString = "& { set-itemproperty -path '$key' -name '$regName' -Type $dataType -Value $data }" + RunPowershellCommand -command "$commandString" -elevated $elevated -maxErrorLevel 0 + + + #if ($elevated) { + # $commandString = "& { set-itemproperty -path '$key' -name '$regName' -Type $dataType -Value $data }" + # RunPowershellCommand -command "$commandString" -elevated $true -maxErrorLevel 0 + #} + #else { + # set-itemproperty -path '$key' -name '$regName' -Type $dataType -Value $data + #} + } +} + +function CreateBuildProtobufBatch( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $filename = $table["FileName"] + $sourceDir = $table["SourceDir"] + $targetDir = $table["TargetDir"] + + if ($Execute) { + Remove-Item -Path $filename -ErrorAction SilentlyContinue | Out-Null + + $batchScript = GetBatchBuildProtoBuf $sourceDir $targetDir + + add-content -Path $filename -Encoding Ascii -Value $batchScript + } +} + +function CreateBuildZlibBatch( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $filename = $table["FileName"] + $zlibSourceDir = $table["zlibSourceDir"] + $libzipSourceDir = $table["libzipSourceDir"] + $targetDir = $table["TargetDir"] + + if ($Execute) { + Remove-Item -Path $filename -ErrorAction SilentlyContinue | Out-Null + + $batchScript = GetBatchBuildZlibBuf $zlibSourceDir $libzipSourceDir $targetDir + + add-content -Path $filename -Encoding Ascii -Value $batchScript + } +} + +function DoProcess( + [bool] $doExecute = $false, + [string] $command, + [string] $param, + [string] $workingDir = "", + [boolean] $requiresRunAs = $false, + [int] $maxErrorLevel = 0, + [bool] $throwOnError = $true) +{ + Write-Verbose "start-process [$command] with [$param]" + + if (-not $DoExecute) { + Write-Host "DEMOMODE - setting Exit Code: 0" + return + } + + if ($workingDir.Length -eq 0) { + if ($requiresRunAs) { + $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru -Verb runas + } + else { + $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru + } + + } + else { + if ($requiresRunAs) { + $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru -Verb runas -WorkingDirectory "$workingDir" + } + else { + $process = start-process -FilePath "$command" -ArgumentList "$param" -Wait -PassThru -WorkingDirectory "$workingDir" + } + } + + $eCode = ($process.ExitCode) + + if (-not $throwOnError) { + if ($ecode -gt $maxErrorLevel) { + Write-Verbose "Running [start-process $commandString $param] failed with exit code [$ecode]" + return + } + return + } + + if ($ecode -gt $maxErrorLevel) { + throw "Running [start-process $commandString $param] failed with exit code [$ecode]" + } + return +} + +function SetEnvVar( + [Parameter(Mandatory=$true)][string] $name, + [Parameter(Mandatory=$true)][string] $content, + [string] $location = "Machine") +{ + Write-Verbose "SetEnvVar [$name] with [$content]" + + if ($Execute) { + $commandString = "& { [environment]::SetEnvironmentVariable('"+$name+"', '"+$content+"', '"+$location+"') }" + RunPowershellCommand -command "$commandString" -elevated $true -maxErrorLevel 0 + } +} + +function RunPowershellCommand( + [string] $commandString, + [boolean] $elevated, + [int] $maxErrorLevel) +{ + $commandBytes = [System.Text.Encoding]::Unicode.GetBytes($commandString) + $encodedCommand = [Convert]::ToBase64String($commandBytes) + $commandLine = "-NoProfile -EncodedCommand $encodedCommand" + + if ($elevated) { + $process = Start-Process -PassThru -FilePath powershell.exe -ArgumentList $commandLine -wait -verb runas + } + else { + $process = Start-Process -PassThru -FilePath powershell.exe -ArgumentList $commandLine -wait + } + $eCode = ($process.ExitCode) + if ($ecode -gt $maxErrorLevel) { + throw "Running 'powershell.exe $commandString' failed with exit code [$ecode]" + } + return +} + +function Invoke-DosCommand { + [CmdletBinding()] + Param( + [ValidateScript({ Get-Command $_ })] + [string] $Command, + [string[]] $Argument, + [string] [ValidateScript({ Test-Path -PathType Container $_ })] $WorkingDirectory, + [int] $maxErrorLevel, + [switch] $SuppressOutput + ) + Write-Verbose "Running '$Command $Argument'" + if ($WorkingDirectory) { + Push-Location $WorkingDirectory -ErrorAction Stop + } + if ($SuppressOutput) { + $null = & $Command $Argument 2>&1 + } else { + & $Command $Argument + } + if ($WorkingDirectory) { + Pop-Location + } + if ($LASTEXITCODE -gt $maxErrorLevel) { + throw "Running DOS Command '$Command $Argument' failed with exit code $LASTEXITCODE" + } +} + +function ResolveApplicationName( + [string] $name, + [string] $directory, + [bool] $usePath) +{ + $application = "" + + if ($directory.Length -gt 0) { + $application = CallGetCommand (join-path $directory $name) + } + if ($application.Length -eq 0) { + if ($usePath) { + # we are at this point if we are supposed to check in the path environment for a match and + # $directory was empty or we couldn't find it in the $directory + + $application = CallGetCommand $name + } + } + # application will be an empty string if we couldn't resolve the name, otherwise we can execute $application + + return $application +} + +function CallGetCommand( + [string] $application) +{ + try { + get-command $application -CommandType Application -ErrorAction Stop | Out-Null + return $application + } + catch { + # the application can't be found, so return empty string + return "" + } +} + +function GetBatchBuildProtoBuf( + [string] $sourceDir, + [string] $targetDir) +{ + $batchscript = @" +@SET VCDIRECTORY=C:\Program Files (x86)\Microsoft Visual Studio 14.0 +@SET SOURCEDIR=$sourceDir +@SET TARGETDIR=$targetDir + +@echo. +@echo This will build Protobuf-3.1.0 +@echo ------------------------------ +@echo The configured settings for the batch file: +@echo Visual Studio directory: %VCDIRECTORY% +@echo Protobuf source directory: %SOURCEDIR% +@echo Protobuf target directory: %TARGETDIR% +@echo. +@echo. +@echo Please edit the batch file if this doesn't match your directory layout! +@echo. + +@pause + +@call "%VCDIRECTORY%\VC\vcvarsall.bat" amd64 + +@pushd %SOURCEDIR% +@cd cmake +@md build && cd build + +@md debug && cd debug +@cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Debug -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" ..\.. +@nmake +@nmake install +@cd .. + +@md release && cd release +@cmake -G "NMake Makefiles" -DCMAKE_BUILD_TYPE=Release -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_MSVC_STATIC_RUNTIME=OFF -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" ..\.. +@nmake +@nmake install +@cd .. + +@popd + +setx PROTOBUF_PATH %TARGETDIR% +"@ + + return $batchscript +} + +function GetBatchBuildZlibBuf( + [string] $zlibSourceDir, + [string] $libzipSourceDir, + [string] $targetDir) +{ + $batchscript = @" +@SET VCDIRECTORY=C:\Program Files (x86)\Microsoft Visual Studio 14.0 +@SET LIBZIPSOURCEDIR=$libzipSourceDir +@SET ZLIBSOURCEDIR=$zlibSourceDir +@SET TARGETDIR=$targetDir +@SET CMAKEGEN="Visual Studio 14 2015 Win64" + +@echo. +@echo This will build ZLib using Visual Studio 2015 +@echo --------------------------------------------- +@echo The configured settings for the batch file: +@echo Visual Studio directory: %VCDIRECTORY% +@echo CMake Generator: %CMAKEGEN% +@echo LibZip source directory: %LIBZIPSOURCEDIR% +@echo Zlib source directory: %ZLIBSOURCEDIR% +@echo Zlib-VS15 target directory: %TARGETDIR% +@echo. +@echo. +@echo Please edit the batch file if this doesn't match your directory layout! +@echo. + +@pause + +@call "%VCDIRECTORY%\VC\vcvarsall.bat" amd64 + +@pushd %ZLIBSOURCEDIR% +@mkdir build +@cd build +@cmake .. -G%CMAKEGEN% -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" +@msbuild /P:Configuration=Release INSTALL.vcxproj +@popd + +@pushd %LIBZIPSOURCEDIR% +@md build +@cd build +@cmake .. -G%CMAKEGEN% -DCMAKE_INSTALL_PREFIX="%TARGETDIR%" +@msbuild libzip.sln /t:zip /P:Configuration=Release +@cmake -DBUILD_TYPE=Release -P cmake_install.cmake +@popd + +setx ZLIB_PATH %TARGETDIR% +"@ + + return $batchscript +} + + + +function GetCygwinBashScript +{ + $batchscript = @" +easy_install-2.7 pip +pip install six +pip install pytest +"@ + + return $batchscript +} + + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Common.ps1 b/Scripts/devInstall/Windows/helper/Common.ps1 new file mode 100644 index 000000000..348b16c95 --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Common.ps1 @@ -0,0 +1,42 @@ +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# + +function GetTableDefaultBool( + [hashtable] $table, + [string] $entryName, + [bool] $defaultValue +) +{ + if ($table[$entryName] -eq $null) { + return $defaultValue + } + return $table[$entryName] +} + +function GetTableDefaultString( + [hashtable] $table, + [string] $entryName, + [string] $defaultValue +) +{ + if ($table[$entryName] -eq $null) { + return $defaultValue + } + return $table[$entryName] +} + +function GetTableDefaultInt( + [hashtable] $table, + [string] $entryName, + [int] $defaultValue +) +{ + if ($table[$entryName] -eq $null) { + return $defaultValue + } + return $table[$entryName] +} + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Display.ps1 b/Scripts/devInstall/Windows/helper/Display.ps1 new file mode 100644 index 000000000..49f2c5c93 --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Display.ps1 @@ -0,0 +1,149 @@ +function FunctionIntro( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + $table | Out-String | Write-Verbose +} + +function GetKey( + [string] $validChar +) +{ + do { + $key = Read-Host + } until ($key -match $validChar) + + return $key +} + +function DisplayStartMessage +{ +" + +This script will setup the CNTK v2 Development Environment on your machine. +More help is given by calling get-help .\devInstall.ps1 + +The script will analyse your machine and will determine which components are required. +The required components will be downloaded into [$localCache] +Repeated operation of this script will reuse already downloaded components. +" +} + +function DisplayVersionWarningMessage( + [string] $version) +{ +" +You are executing this script from Powershell Version $version. +We recommend that you execute the script from Powershell Version 4 or later. You can install Powershell Version 4 from: + https://www.microsoft.com/en-us/download/details.aspx?id=40855 +" +} + +function DisplayWarningNoExecuteMessage +{ +" +The parameter '-Execute' hasn't be supplied to the script. +The script will execute withouth making any actual changes to the machine! +" +} + +function DisplayStartContinueMessage +{ +" +1 - I agree and want to continue +Q - Quit the installation process +" +} + +function CheckPowershellVersion +{ + $psVersion = $PSVersionTable.PSVersion.Major + if ($psVersion -ge 4) { + return $true + } + + Write-Warning $(DisplayVersionWarningMessage $psVersion) + if ($psVersion -eq 3) { + return $true + } + return $false +} + +function CheckOSVersion +{ + $runningOn = (Get-WmiObject -class Win32_OperatingSystem).Caption + $isMatching = ($runningOn -match "^Microsoft Windows (8\.1|10|Server 2012 R2|Server 2016)") + + if (-not $isMatching) { + Write-Warning " +You are running the this install script on [$runningOn]. +The Microsoft Cognitive Toolkit is designed and tested on Windows 8.1, Windows 10, +Windows Server 2012 R2, and Windows Server 2016. +" + } +} + +function DisplayStart() +{ + Write-Host $(DisplayStartMessage) + + if (-not (CheckPowershellVersion)) { + return $false + } + + CheckOSVersion + + if (-not $Execute) { + Write-Warning $(DisplayWarningNoExecuteMessage) + } + + Write-Host $(DisplayStartContinueMessage) + $choice = GetKey '^[1qQ]+$' + + if ($choice -contains "1") { + return $true + } + + return $false +} + + +Function DisplayEnd() +{ + +} + +function DisplayAfterVerify( + [array] $list = @()) +{ + Write-Host + + if ($list.Count -gt 0) { + Write-Host "The following operations will be performed:" + + foreach ($item in $list) { + $info = $item.ActionInfo + Write-Host " * $info" + } + if (-not $Execute) { + Write-Warning $(DisplayWarningNoExecuteMessage) + } + + Write-Host + Write-Host "Do you want to continue? (y/n)" + + $choice = GetKey '^[yYnN]+$' + + if ($choice -contains "y") { + return $true + } + } + else { + Write-Host "No additional installation required" + Write-Host + + } + return $false +} + +# vim:set expandtab shiftwidth=2 tabstop=2: diff --git a/Scripts/devInstall/Windows/helper/Download.ps1 b/Scripts/devInstall/Windows/helper/Download.ps1 new file mode 100644 index 000000000..6bc4c6771 --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Download.ps1 @@ -0,0 +1,298 @@ +function DownloadOperations( + [parameter(Mandatory=$true)][array] $downloadList) +{ + Write-Host "Performing download operations" + + foreach ($item in $downloadList) { + foreach ($downloadItem in $item.Download) { + DownloadItem $downloadItem + } + } + + Write-Host "Download operations finished" + Write-Host +} + + +function DownloadItem( + [hashtable] $item +) +{ + $func = $item["Function"] + + $expr = $func +' $item' + + Write-Verbose "Calling Operation: [$func]" + $result = Invoke-Expression $expr + + return +} + +function DownloadForPlatform( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $platform = $table["Platform"] + + if (PlatformMatching $platform) { + Download $table + } +} + +function Download( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + + $func = $table["Function"] + $source = $table["Source"] + $method = GetTableDefaultString -table $table -entryName "Method" -defaultValue "WebRequest" + $destination = $table["Destination"] + $ExpectedSize = GetTableDefaultInt -table $table -entryName "ExpectedSize" -defaultValue 0 + + if (test-path $destination -PathType Leaf) { + Write-Host File [$destination] already exists + return + } + + if ($method -eq "WebRequest") { + DownloadFileWebRequest -SourceFile $source -OutFile $destination -ExpectedSize $ExpectedSize + } + else { + DownloadFile -SourceFile $source -OutFile $destination + } + return +} + +function LocalCopyFile( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $source = $table["Source"] + $destination = $table["Destination"] + + if (-not $Execute) { + Write-Host "$message ** Running in DEMOMODE - no download performed" + return $true + } + if (test-path $destination -PathType Leaf) { + Write-Host File [$destination] already exists + return + } + if (-not (test-path $source -PathType Leaf)) { + throw "Sourcefile [$source] is missing" + } + + Write-Host Copying [$source] to local disk ... + new-item $destination -type File -Force -ErrorAction SilentlyContinue + copy-Item $source $destination -Force -ErrorAction SilentlyContinue + return +} + +function RobocopyFromServer( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + + $source = $table["Source"] + $destination = $table["Destination"] + + $source = $LocalServer+"\$source" + + if (-not (test-path $source)) { + throw "SourceDirectory [$source] is missing" + } + + $option = "/MIR /NFL /copy:DT /dcopy:D /xj" + + $param = "$source $destination $option" + + DoProcess -command $roboCopyCmd -param "$param" -IgnoreNonZeroExitCode $true +} + +function NotImplemented( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + throw "Call to function 'NotImplemented' " +} + +function DownloadAndExtract( + [string] $tPath, + [string] $sAddress, + [string] $fileName, + [string] $targetPathRoot +){ + $outFileName = Join-Path $tPath $fileName + + DownloadFile -SourceFile $sAddress ` + -OutFile $outFileName ` + -tempFileName $fileName + + Write-Host Extracting into targetpath + Write-Host + + ExtractAllFromZip $outFileName $targetPathRoot +} + + +function DownloadFileWebRequest ( + [string] $SourceFile, + [string] $OutFile, + [int] $ExpectedSize) +{ + Write-Host "Downloading [$SourceFile], please be patient...." + if (-not $Execute) { + Write-Host "$message ** Running in DEMOMODE - no download performed" + return $true + } + + if (test-path -path $outFile) { + Write-Host "File [$outFile] already exists" + return $true + } + + $TempFile = [System.IO.Path]::GetTempFileName() + try { + $response = Invoke-WebRequest -Uri $SourceFile -OutFile $TempFile -TimeoutSec 120 + } + catch { + $errorCode = $_.Exception.Response.StatusCode.Value__ + + Remove-Item -path $TempFile -ErrorAction SilentlyContinue + throw "Download $SourceFile Failed! WebRequest reported errorCode: [$errorCode]" + } + + # we now have the temporary file + if (($ExpectedSize -gt 0) -and (Test-Path $TempFile)) { + $fileSize = (Get-Item $TempFile).length + if (-not ($fileSize -eq $ExpectedSize)) { + Remove-Item -path $TempFile -ErrorAction SilentlyContinue + throw "Size of downloaded file [$fileSize] not matching expected filesize [$ExpectedSize] for $sourceFile" + } + } + + new-item $outFile -type File -Force -ErrorAction SilentlyContinue + move-Item $TempFile $OutFile -Force -ErrorAction SilentlyContinue + + if (-not (Test-Path -Path $OutFile)) { + throw "Download $SourceFile Failed!" + } + # we have a file with our expected filename, we are in good shape and ready to report success + # in case the above rename failed, but the target file exist, we clean up a possible dangling TempFile + # no need to check if this file is really there. we don't care if it succeeds or not + Remove-Item -path $TempFile -ErrorAction SilentlyContinue +} + +function DownloadFile( + [string] $SourceFile, + [string] $OutFile, + [int] $timeout = 600, + [int] $maxtry = 5) +{ + # the scriptblock to invoke a web-request + $sb ={ + param([string]$uri,[string]$outfile) + (New-Object System.Net.WebClient).DownloadFile($uri,$outfile) + } + #--------------- + + $startTime = Get-Date + Write-Host "Downloading [$SourceFile], please be patient...." + if (-not $Execute) { + Write-Host "$message ** Running in DEMOMODE - no download performed" + return + } + + $TempFile = [System.IO.Path]::GetTempFileName() + + for ($count=1; $count -le $maxtry; $count +=1) { + if ($count -gt 1) { + Write-Host "Iteration [$count] of [$maxtry]" + } + + if ($count -gt 1) { + start-sleep -Seconds 5 + } + if (Test-Path -Path $TempFile) { + # the file we temporary use as a download cache could exist + # if it does, we remove it and terminate if this fails + Remove-Item -path $TempFile -ErrorAction Stop + } + + if (test-path -path $outFile) { + Write-Host "File [$outFile] already exists" + return + } + + $job = start-job -scriptblock $sb -ArgumentList $sourceFile, $TempFile + Wait-Job $job -Timeout $timeout + + $jState = $job.State.ToUpper() + $jStart = $job.PSBeginTime + $jEnd = $job.PSEndTime + $jError = $job.ChildJobs[0].Error + $current = Get-Date + + switch ($jState) { + "COMPLETED" { + if ($jError.Count -eq 0) { + Write-Verbose "End binary download!" + + Remove-Job $job -force -ErrorAction SilentlyContinue + + # we now have the temporary file, we need to rename it + new-item $outFile -type File -Force -ErrorAction SilentlyContinue + move-Item $TempFile $OutFile -Force -ErrorAction SilentlyContinue + + if (Test-Path -Path $OutFile) { + # we have a file with our expected filename, we are in good shape and ready to report success + # in case the above rename failed, but the target file exist, we clean up a possible dangling TempFile + # no need to check if this file is really there. we don't care if it succeeds or not + Remove-Item -path $TempFile -ErrorAction SilentlyContinue + + return + } + + # we got here because we finished the job, but some operation failed (i.e. the rename above. we can just try again) + Write-Verbose "Job completed but rename operation failed, retrying..." + continue + } + + Write-Host "Job Completed with Error: [$jStart] to [$current]" + Write-Host $jError + } + "RUNNING" { + Write-Host "Job Timeout: [$jStart] to [$current]" + } + "FAILED" { + $current = Get-Date + Write-Host "Job Failed: [$jStart] to [$current]" + Write-Host "Error: $jError" + } + default { + Write-Host "Job State: [$Error] - [$jStart] to [$current]" + } + } + Remove-Job $job -force -ErrorAction SilentlyContinue + } + + throw "Download $SourceFile Failed!" +} + +function PlatformMatching( + [string] $regExprPlatform) +{ + $runningOn = ((Get-WmiObject -class Win32_OperatingSystem).Caption).ToUpper() + $isMatching = ($runningOn -match $regExprPlatform) + + Write-Verbose "Function [PlatformMatching]: $runningOn -match on platform [$regExprPlatform] = [$isMatching]" + return $isMatching +} + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Operations.ps1 b/Scripts/devInstall/Windows/helper/Operations.ps1 new file mode 100644 index 000000000..f2980af24 --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Operations.ps1 @@ -0,0 +1,662 @@ +function OpAnaconda3411( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "Anaconda3-4.1.1" + $prodFile = "Anaconda3-4.1.1-Windows-x86_64.exe" + $prodSubDir = "Anaconda3-4.1.1-Windows-x86_64" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "https://repo.continuum.io/archive/Anaconda3-4.1.1-Windows-x86_64.exe" + $downloadSize = 370055720 + + @( @{ShortName = "ANA3-411"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath; } ); + Download = @( @{Function = "Download"; Method = "WebRequest"; Source = $downloadSource; Destination = "$cache\$prodFile"; ExpectedSize = $downloadSize } ); + Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/InstallationType=JustMe /AddToPath=0 /RegisterPython=0 /S /D=$targetPath"; runAs=$false; Message = ".... This will take some time. Please be patient ...." } ); + } ) +} + +function OpAnacondaEnv34( + [parameter(Mandatory=$true)][string] $targetFolder, + [parameter(Mandatory=$true)][string] $repoDir, + [parameter(Mandatory=$true)][string] $reponame) +{ + $prodName = "Python 3.4 Environment" + + $prodSubDir = "Anaconda3-4.1.1-Windows-x86_64" + $targetPath = join-path $targetFolder $prodSubDir + $envName = "cntkdev-py34" + $envDir = "envs\$envName" + $envVar = "CNTK_PY34_PATH"; + $envValue = join-path $targetPath $envDir + + $ymlDirectory = join-path $repoDir $repoName + $ymlDirectory = join-path $ymlDirectory "scripts\install\windows" + + @{ ShortName = "PYENV34"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Creating $prodName"; + Verification = @( @{Function = "VerifyRunAlways" } ); + Action = @( @{Function = "InstallYml"; BasePath = $targetPath; Env = $envName; ymlFile= "$ymlDirectory\conda-windows-cntk-py34-environment.yml" }, + @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ) + } +} + +function OpBoost160( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "Boost 1.60.0" + $prodFile = "boost_1_60_0-msvc-12.0-64.exe" + $prodSubDir = "boost_1_60_0" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "https://sourceforge.net/projects/boost/files/boost-binaries/1.60.0/boost_1_60_0-msvc-12.0-64.exe/download" + + @( @{Name = $prodName; ShortName = "BOOST160"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = "BOOST_INCLUDE_PATH"; Content = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = "BOOST_LIB_PATH"; Content = "$targetPath\lib64-msvc-12.0" } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "SetEnvironmentVariable"; EnvVar = "BOOST_INCLUDE_PATH"; Content = "$targetPath" }, + @{Function = "SetEnvironmentVariable"; EnvVar = "BOOST_LIB_PATH"; Content = "$targetPath\lib64-msvc-12.0" }, + @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/dir=$targetPath /SP- /SILENT /NORESTART"; runAs=$false } ); + } ) +} + +function OpBoost160VS15( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "Boost 1.60.0" + $prodFile = "boost_1_60_0-msvc-14.0-64.exe" + $prodSubDir = "boost_1_60_0-msvc-14.0" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "https://sourceforge.net/projects/boost/files/boost-binaries/1.60.0/boost_1_60_0-msvc-14.0-64.exe/download" + $envVar = "BOOST_INCLUDE_PATH" + $envVarLib = "BOOST_LIB_PATH" + $envContentLib = "$targetPath\lib64-msvc-14.0" + + @( @{Name = $prodName; ShortName = "BOOST160VS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVarLib; Content = $envContentLib } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $targetPath }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVarLib; Content = $envContentLib }, + @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/dir=$targetPath /SP- /SILENT /NORESTART"; runAs=$false } ); + } ) +} + +function OpCMake362( + [parameter(Mandatory=$true)][string] $cache) +{ + $prodName = "CMake 3.6.2" + $targetPath = join-path $env:ProgramFiles "cmake\bin" + $cmakeName = "cmake-3.6.2-win64-x64.msi" + $downloadSource = "https://cmake.org/files/v3.6/cmake-3.6.2-win64-x64.msi" + + @( @{ShortName = "CMake362"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyWinProductExists"; Match = "^CMake$"; Version = "3.6.2" } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$cmakeName" } ); + Action = @( @{Function = "InstallMsi"; MsiName = "$cmakeName" ; MsiDir = "$cache" } , + @{Function = "AddToPath"; Dir = "$targetPath" } ); + } ) +} + +function OpCNTKMKL2 + ([parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "CNTK Custom MKL Version 2" + $prodFile = "CNTKCustomMKL-Windows-2.zip" + $prodSubDir = "CNTKCustomMKL" + $targetPath = join-path $targetFolder $prodSubDir + $targetPathCurrenVersion = join-path $targetPath "2" + $envVar = "CNTK_MKL_PATH"; + $envValue = $targetPath + $downloadSource = "https://www.cntk.ai/mkl/$prodFile"; + + @( @{ShortName = "CNTKMKL2"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPathCurrenVersion"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPathCurrenVersion }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $localDir; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); + } ) +} + +function OpCNTKMKL3 + ([parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "CNTK Custom MKL Version 3" + $prodFile = "CNTKCustomMKL-Windows-3.zip" + $prodSubDir = "CNTKCustomMKL" + $targetPath = join-path $targetFolder $prodSubDir + $targetPathCurrenVersion = join-path $targetPath "3" + $envVar = "CNTK_MKL_PATH"; + $envValue = $targetPath + $downloadSource = "https://www.cntk.ai/mkl/$prodFile"; + + @( @{ShortName = "CNTKMKL3"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPathCurrenVersion"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPathCurrenVersion }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $localDir; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); + } ) +} + +function OpCygwin + ([parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "CygWin64" + $prodFile = "cygwinSetup_64.exe" + $prodSubDir = "cygwin64" + $targetPath = join-path $targetFolder $prodSubDir + $regkey = "Registry::HKEY_CURRENT_USER\SOFTWARE\Cygwin" + $downloadSource = "https://www.cygwin.com/setup-x86_64.exe" + $installParam = "--quiet-mode --no-admin --site http://mirrors.kernel.org/sourceware/cygwin/ --root $targetPath --local-package-dir $cache --packages python,python-yaml,python-numpy,diffutils" + $bFileName = "cygwinpip.bash" + $bashFileName = join-path $targetFolder $bFileName + $bashParmFile = Join-Path "\cygdrive\c" (split-path $bashFileName -NoQualifier) + $bashParmFile = $bashParmFile.replace("\","/") + $bashParam = "-l -c $bashParmFile" + + @( @{ShortName = "CYGWIN64"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" }, + @{Function = "VerifyRegistryKey"; Key = $regKey } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = $installParam; runAs=$false; WorkDir = $targetFolder}, + @{Function = "AddToPath"; Dir = "$targetPath\bin"; AtStart = $false }, + @{Function = "CreateCygwinBashScript"; FileName = $bashFileName }, + @{Function = "InstallExe"; Command = "$targetPath\bin\bash.exe"; Param = $bashParam; runAs=$false; WorkDir = $targetPath } ); + } ) +} + +function AddOpDisableJITDebug +{ + @( @{Name="Visual Studio Disable Just-In-Time Debugging"; ShortName = "JITVS2013"; VerifyInfo = "Checking for JIT Debugging registry keys"; ActionInfo = "Removing registry keys to disable JIT debugging, advisable for Jenkins machines"; + Verification = @( @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug"; RegName = "Debugger"; }, + @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework"; RegName = "DbgManagedDebugger" }, + @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Windows NT\CurrentVersion\AeDebug"; RegName = "Debugger" }, + @{Function = "DeletedRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\.NETFramework"; RegName = "DbgManagedDebugger" } ); + Action = @( @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows NT\CurrentVersion\AeDebug"; Elevated = $true; RegName = "Debugger" }, + @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\.NETFramework"; Elevated = $true; RegName = "DbgManagedDebugger" }, + @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\Windows NT\CurrentVersion\AeDebug"; Elevated = $true; RegName = "Debugger" }, + @{Function = "RemoveRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\Wow6432Node\Microsoft\.NETFramework"; Elevated = $true; RegName = "DbgManagedDebugger" } ) + } ) + } + +function OpGit2101( + [parameter(Mandatory=$true)][string] $cache) +{ + $prodName = "Git" + $targetPath = join-path $env:ProgramFiles "Git\bin" + $prodFile = "Git-2.10.1-64-bit.exe" + $downloadSource = "https://github.com/git-for-windows/git/releases/download/v2.10.1.windows.1/$prodFile" + + @( @{ShortName = "GIT"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyRegistryKeyName"; Key = "Registry::HKEY_LOCAL_MACHINE\SOFTWARE\GitForWindows"; RegName = "CurrentVersion"; } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "InstallExe"; Command = "$cache\$prodFile"; Param = "/SP- /SILENT /NORESTART"}, + @{Function = "AddToPath"; Dir = $targetPath; AtStart = $true; } ) + } ) +} + +function OpGitClone( + [parameter(Mandatory=$true)][string] $targetFolder, + [parameter(Mandatory=$true)][string] $targetDir, + [string] $repoTag = "master") +{ + $targetPath = join-path $targetFolder $targetDir + $downloadSource = "https://github.com/Microsoft/CNTK/"; + $appDir = join-path $env:ProgramFiles "Git\bin" + + @( @{Name = "Clone CNTK from Github"; ShortName = "CNTKCLONE"; VerifyInfo = "Checking for CNTK-Clone target directory $targetPath"; ActionInfo = "Cloneing CNTK from Github repository"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); + Action = @( @{Function = "MakeDirectory"; Path = $targetFolder }, + @{Function = "ExecuteApplication"; AppName = "git.exe"; Param = "clone --branch $repoTag --recursive https://github.com/Microsoft/CNTK/"; AppDir = $appDir; UseEnvPath = $true; WorkDir = $targetFolder } ) + } ) +} + +function OpMSMPI70([parameter( + Mandatory=$true)][string] $cache) +{ + $remoteFilename = "MSMpiSetup.exe" + $localFilename = "MSMpiSetup70.exe" + $downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename"; + + @( @{Name = "MSMPI Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70"; ActionInfo = "Installing MSMPI 70"; + Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft MPI \(\d+\."; Version = "7.0.12437.6"; MatchExact = $false } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$localFilename" } ); + Action = @( @{Function = "InstallExe"; Command = "$cache\$localFilename" ; Param = "/unattend" } ) + } ) +} + +function OpMSMPI70SDK( + [parameter(Mandatory=$true)][string] $cache) +{ + $remoteFilename = "msmpisdk.msi" + $localFilename = "msmpisdk70.msi" + $downloadSource = "https://download.microsoft.com/download/D/7/B/D7BBA00F-71B7-436B-80BC-4D22F2EE9862/$remoteFilename"; + + @( @{Name = "MSMPI SDK70 Installation"; ShortName = "CNTK"; VerifyInfo = "Checking for installed MSMPI 70 SDK"; ActionInfo = "Install MSMPI 70 SDK"; + Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft MPI SDK \(\d+\."; Version = "7.0.12437.6"; MatchExact = $false } ); + #Verification = @( @{Function = "VerifyWinProductExists"; Match = "^Microsoft MPI SDK \(\d+\."; Compare = "^Microsoft MPI SDK \(7\.0\.12437\.6\)"; MatchExact = $false } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$localFilename" } ); + Action = @( @{Function = "InstallMsi"; MsiName = "$localFilename" ; MsiDir = "$cache" } ) + } ) +} + +function OpNvidiaCub141( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "NVidia CUB 1.4.1" + $prodFile = "cub-1.4.1.zip" + $prodSubDir = "cub-1.4.1" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "CUB_PATH"; + $envValue = $targetPath + $downloadSource = "https://codeload.github.com/NVlabs/cub/zip/1.4.1"; + + @( @{ShortName = "CUB141"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = "$targetFolder"; destinationFolder = $prodSubDir; zipSubTree= $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); + } ) +} + +function OpNVidiaCuda75( + [parameter(Mandatory=$true)][string] $cache) +{ + $cudaDownload = "http://developer.download.nvidia.com/compute/cuda/7.5/Prod/local_installers" + $programPath = join-path $env:ProgramFiles "NVIDIA GPU Computing Toolkit\CUDA\v7.5" + + + @( @{Name="NVidia CUDA 7.5"; ShortName = "CUDA75"; VerifyInfo = "Checking for installed NVidia Cuda 75"; ActionInfo = "Installing CUDA 7.5"; + Verification = @( @{Function = "VerifyDirectory"; Path = $programPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = "CUDA_PATH_V7_5"; Content = $programPath } ); + Download = @( @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Source = "$cudaDownload/cuda_7.5.18_windows.exe"; Destination = "$cache\cuda_7.5.18_windows.exe" }, + @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows 10"; Source = "$cudaDownload/cuda_7.5.18_win10.exe"; Destination = "$cache\cuda_7.5.18_win10.exe" } ); + Action = @( @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Command = "$cache\cuda_7.5.18_windows.exe"; Param = "-s CUDAToolkit_7.5 CUDAVisualStudioIntegration_7.5 GDK"; Message = ".... This will take some time. Please be patient ...." } , + @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows 10"; Command = "$cache\cuda_7.5.18_win10.exe"; Param = "-s CUDAToolkit_7.5 CUDAVisualStudioIntegration_7.5 GDK"; Message = ".... This will take some time. Please be patient ...." } ); + }) +} + +function OpNVidiaCuda80( + [parameter(Mandatory=$true)][string] $cache) +{ + $cudaDownload = "http://developer.download.nvidia.com/compute/cuda/8.0/secure/prod/local_installers" + $cudaFile = "cuda_8.0.44_windows.exe" + $cudaFileWin10 = "cuda_8.0.44_win10.exe" + $programPath = join-path $env:ProgramFiles "NVIDIA GPU Computing Toolkit\CUDA\v8.0" + $cudaInstallParam = "-s" + + @( @{Name="NVidia CUDA 8.0"; ShortName = "CUDA80"; VerifyInfo = "Checking for installed NVidia Cuda 80"; ActionInfo = "Installing CUDA 8.0"; + Verification = @( @{Function = "VerifyDirectory"; Path = $programPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = "CUDA_PATH_V8_0"; Content = $programPath } ); + Download = @( @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Source = "$cudaDownload/$cudaFile"; Destination = "$cache\$cudaFile" }, + @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows 10"; Source = "$cudaDownload/$cudaFileWin10"; Destination = "$cache\$cudaFileWin10" } ); + Action = @( @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows (7|8|Server 2008 R2|Server 2012 R2)"; Command = "$cache\$cudaFile"; Param = $cudaInstallParam } , + @{Function = "InstallExeForPlatform"; Platform = "^Microsoft Windows 10"; Command = "$cache\$cudaFileWin10"; Param = $cudaInstallParam } ); + }) +} + +function OpNVidiaCudnn5175( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) + +{ + $prodName = "NVidia CUDNN 5.1 for CUDA 7.5" + $cudnnWin7 = "cudnn-7.5-windows7-x64-v5.1.zip" + $cudnnWin10 = "cudnn-7.5-windows10-x64-v5.1.zip" + + $prodSubDir = "cudnn-7.5-v5.1" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "CUDNN_PATH" + $envValue = join-path $targetPath "cuda" + $downloadSource = "http://developer.download.nvidia.com/compute/redist/cudnn/v5.1" + + @( @{ShortName = "CUDNN5175"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows 7"; Source = "$downloadSource/$cudnnWin7"; Destination = "$cache\$cudnnWin7" }, + @{Function = "DownloadForPLatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; Source = "$downloadSource/$cudnnWin10"; Destination = "$cache\$cudnnWin10" } ); + Action = @( @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows 7"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, + @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + }) +} + +function OpNVidiaCudnn5180( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "NVidia CUDNN 5.1 for CUDA 8.0" + $cudnnWin7 = "cudnn-8.0-windows7-x64-v5.1.zip" + $cudnnWin10 = "cudnn-8.0-windows10-x64-v5.1.zip" + + $prodSubDir = "cudnn-8.0-v5.1" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "CUDNN_PATH" + $envValue = join-path $targetPath "cuda" + $downloadSource = "http://developer.download.nvidia.com/compute/redist/cudnn/v5.1" + + @( @{ShortName = "CUDNN5180"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyDirectory"; Path = $envValue }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{Function = "DownloadForPlatform"; Platform = "^Microsoft Windows 7"; Source = "$downloadSource/$cudnnWin7"; Destination = "$cache\$cudnnWin7" }, + @{Function = "DownloadForPlatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; Source = "$downloadSource/$cudnnWin10"; Destination = "$cache\$cudnnWin10" } ); + Action = @( @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows 7"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, + @{Function = "ExtractAllFromZipForPlatform"; Platform = "^Microsoft Windows (8|10|Server 2008 R2|Server 2012 R2)"; zipFileName = "$cache\$cudnnWin10"; destination = $targetFolder; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + }) +} + +function OpOpenCV31( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "OpenCV-3.1" + $prodFile = "opencv-3.1.0.exe" + $prodSubDir = "OpenCV310" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "OPENCV_PATH_V31"; + $envValue = "$targetPath\build" + $downloadSource = "https://netcologne.dl.sourceforge.net/project/opencvlibrary/opencv-win/3.1.0/opencv-3.1.0.exe" + + @( @{ShortName = "OPENCV310"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Please perform a manual installation of $prodName from $cache"; + Verification = @( @{Function = "VerifyFile"; Path = "$cache\$prodFile" } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + } ) +} + +function OpOpenCVInternal( + [parameter(Mandatory=$true)][string] $server, + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "OpenCV-3.1" + $prodFile = "opencv-3.1.0.zip" + $prodSubDir = "Opencv3.1.0" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "OPENCV_PATH_V31"; + $envValue = "$targetPath\build" + $downloadSource = "$server\$prodFile" + + @( @{ShortName = "OPENCV310"; Name = $prodName; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = "$targetPath" }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "LocalCopyFile"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = "$targetFolder"; destinationFolder = $prodSubDir; zipSubTree= $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar= $envVar; Content = $envValue } ); + } ) +} + +function OpProtoBuf310VS15( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + # unzip protobuf source in $protoSourceDir = $targetfolder\src\$prodsubdir + # create batch file to build protobuf files in $scriptDirectory = $targetFolder\script + # the script file can be used to create the compiled protobuf libraries in $targetPath = $targetFolder\$prodSubDir + + $prodName = "ProtoBuf 3.1.0 Source" + $prodFile = "protobuf310.zip" + $prodName = "protobuf-3.1.0" + $prodSubDir = "protobuf-3.1.0-vs15" + $batchFile = "buildProto.cmd" + + $protoSourceDir = join-path $targetFolder "src" + $scriptDirectory = join-path $targetFolder "script" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "PROTOBUF_PATH" + $envValue = $targetPath + $downloadSource = "https://github.com/google/protobuf/archive/v3.1.0.zip" + + @( @{ShortName = "PROTO310VS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $protoSourceDir; zipSubTree =$prodName; destinationFolder =$prodName }, + @{Function = "MakeDirectory"; Path = $scriptDirectory }, + @{Function = "CreateBuildProtobufBatch"; FileName = "$scriptDirectory\$batchFile"; SourceDir = (join-path $protoSourceDir $prodName); TargetDir = $targetPath } ); + } ) +} + +function OpProtoBuf310VS15Internal( + [parameter(Mandatory=$true)][string] $server, + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "ProtoBuf 3.1.0 Prebuild VS2015" + $prodFile = "PreBuildProtobuf310vs15.zip" + $prodSubDir = "protobuf-3.1.0-vs15" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "PROTOBUF_PATH" + $envValue = $targetPath + $downloadSource = "$server\$prodFile" + + @( @{ShortName = "PROTO310VS15PRE"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree ="protobuf-3.1.0-vs15"; destinationFolder = $prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + } ) +} + +function OpScanProgram() +{ + @{ ShortName = "SCANPROG"; VerifyInfo = "Scan System for installed programs"; + Verification = @( @{Function = "VerifyScanPrograms" } ) + } +} + +function OpSwig3010( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "SWIG 3.0.10" + $prodFile = "swigwin-3.0.10.zip" + $prodSubDir = "swigwin-3.0.10" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "SWIG_PATH" + $envValue = $targetPath + $downloadSource = "http://prdownloads.sourceforge.net/project/swig/swigwin/swigwin-3.0.10/swigwin-3.0.10.zip" + + @( @{ShortName = "SWIG3010"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree =$prodSubDir; destinationFolder =$prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + } ) +} + +function OpSysinternals( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "Sysinternal Suite" + $prodFile = "SysinternalSuite.zip" + $prodDependsFile = "depends22_x64.zip" + $prodSubDir = "SysInternal" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "https://download.sysinternals.com/files/SysinternalsSuite.zip" + $downloadDependsSource = "http://dependencywalker.com/depends22_x64.zip" + + + @( @{ShortName = "SYSINTERNAL"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName, dependency walker"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyFile"; Path = "$targetPath\depends.exe" }, + @{Function = "VerifyPathIncludes"; Path = $targetPath } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" }, + @{ Function = "Download"; Source = $downloadDependsSource; Destination = "$cache\$prodDependsFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; destinationFolder =$prodSubDir }, + @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodDependsFile"; destination = $targetFolder; destinationFolder =$prodSubDir; AddToDirectory=$true }, + @{Function = "AddToPath"; Dir = $targetPath; AtStart = $false; } + @{Function = "SetRegistryKey"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals" }, + @{Function = "SetRegistryKeyNameData"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals\Autologon"; RegName = "EulaAccepted"; data = 1; dataType = "DWord" }, + @{Function = "SetRegistryKeyNameData"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals\Handle"; RegName = "EulaAccepted"; data = 1; dataType = "DWord" }, + @{Function = "SetRegistryKeyNameData"; Elevated = $true; Key = "registry::HKEY_CURRENT_USER\SOFTWARE\Sysinternals\ProcDump"; RegName = "EulaAccepted"; data = 1; dataType = "DWord" } ) + } ) +} + +function OpTestData( + [parameter(Mandatory=$true)][string] $targetFolder, + [parameter(Mandatory=$true)][string] $remoteData) +{ + $prodName = "Testdata Environment" + @( @{ShortName = "TESTDATA"; VerifyInfo = "Checking for $prodName"; ActionInfo = "Setting up environment variable for $prodName"; + Verification = @( @{Function = "VerifyEnvironment"; EnvVar = "CNTK_EXTERNAL_TESTDATA_REMOTE_DIRECTORY" }, + @{Function = "VerifyEnvironment"; EnvVar = "CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY" } ); + Action = @( @{Function = "SetEnvironmentVariable"; EnvVar = "CNTK_EXTERNAL_TESTDATA_REMOTE_DIRECTORY"; Content = $remoteData }, + @{Function = "SetEnvironmentVariable"; EnvVar = "CNTK_EXTERNAL_TESTDATA_SOURCE_DIRECTORY"; Content = $targetFolder } ); + } ) +} + +function OpAddVS12Runtime([parameter(Mandatory=$true)][string] $cache) +{ + $prodName = "VS2012 Runtime" + + @( @{ShortName = "VS2012"; VerifyInfo = "Checking for $prodName"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2012 x64 Additional Runtime" }, + @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2012 x64 Minimum Runtime" } ); + Download = @( @{ Function = "Download"; Source = "http://download.microsoft.com/download/1/6/B/16B06F60-3B20-4FF2-B699-5E9B7962F9AE/VSU_4/vcredist_x64.exe"; Destination = "$cache\VSRuntime\11\vcredist_x64.exe" } ); + Action = @( @{Function = "InstallExe"; Command = "$cache\VSRuntime\11\vcredist_x64.exe"; Param = "/install /passive /norestart" } ) + } ) +} + +function OpAddVS13Runtime([parameter(Mandatory=$true)][string] $cache) +{ + $prodName = "VS2013 Runtime" + + @( @{ShortName = "VS2013"; VerifyInfo = "Checking for $prodName"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2013 x64 Additional Runtime" }, + @{Function = "VerifyWinProductExists"; Match = "^Microsoft Visual C\+\+ 2013 x64 Minimum Runtime" } ); + Download = @( @{ Function = "Download"; Source = "http://download.microsoft.com/download/2/E/6/2E61CFA4-993B-4DD4-91DA-3737CD5CD6E3/vcredist_x64.exe"; Destination = "$localCache\VSRuntime\12\vcredist_x64.exe" } ); + Action = @( @{Function = "InstallExe"; Command = "$localCache\VSRuntime\12\vcredist_x64.exe"; Param = "/install /passive /norestart" } ) + } ) +} + +function OpCheckVS15Update3 +{ + @( @{Name = "Verify Installation of VS2015, Update 3"; ShortName = "PREVS15U3"; VerifyInfo = "Checking for Visual Studio 2015, Update 3"; + Verification = @( @{Function = "VerifyWinProductVersion"; Match = "^Microsoft Build Tools 14.0 \(amd64\)$"; Version = "14.0.25420"; MatchExact = $true} ); + PreReq = @( @{Function = "PrereqInfoVS15" } ); + Action = @( @{Function = "StopInstallation" } ) + } ) +} + +function OpCheckCuda8 +{ + $programPath = join-path $env:ProgramFiles "NVIDIA GPU Computing Toolkit\CUDA\v8.0" + @( @{Name = "Verify Installation of NVidia Cuda 8"; ShortName = "PRECUDA8"; VerifyInfo = "Checking for NVidia Cuda 8"; + Verification = @( @{Function = "VerifyDirectory"; Path = $programPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = "CUDA_PATH_V8_0"; Content = $programPath } ); + PreReq = @( @{Function = "PrereqInfoCuda8" } ); + Action = @( @{Function = "StopInstallation" } ) + } ) +} + +function OpZlib( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "ZLib128" + $prodFile = "zlib128.zip" + $prodSubDir = "zlib-1.2.8" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "http://zlib.net/zlib128.zip" + + @( @{ShortName = "ZLIN128"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree =$prodSubDir; destinationFolder =$prodSubDir } ); + } ) +} + +function OpLibzip( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "Libzip-1.1.3" + $prodFile = "libzip-1.1.3.tar.gz" + $prodSubDir = "libzip-1.1.3" + $targetPath = join-path $targetFolder $prodSubDir + $downloadSource = "https://nih.at/libzip/libzip-1.1.3.tar.gz" + + @( @{ShortName = "LIBZIP113"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; zipSubTree =$prodSubDir; destinationFolder =$prodSubDir } ); + } ) +} + + +function OpZLibInternal( + [parameter(Mandatory=$true)][string] $server, + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + $prodName = "ZLib Precompiled" + $prodFile = "zlib.zip" + $prodSubDir = "zlib" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "ZLIB_PATH" + $envValue = $targetPath + $downloadSource = "$server\$prodFile" + + @( @{ShortName = "LIBZIP113"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = $targetPath }, + @{Function = "VerifyEnvironmentAndData"; EnvVar = $envVar; Content = $envValue } ); + Download = @( @{ Function = "Download"; Source = $downloadSource; Destination = "$cache\$prodFile" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$prodFile"; destination = $targetFolder; destinationFolder =$prodSubDir }, + @{Function = "SetEnvironmentVariable"; EnvVar = $envVar; Content = $envValue } ); + } ) +} + +function OpZlibVS15( + [parameter(Mandatory=$true)][string] $cache, + [parameter(Mandatory=$true)][string] $targetFolder) +{ + # unzip protobuf source in $protoSourceDir = $targetfolder\src\$prodsubdir + # create batch file to build protobuf files in $scriptDirectory = $targetFolder\script + # the script file can be used to create the compiled protobuf libraries in $targetPath = $targetFolder\$prodSubDir + + $prodName = "zlib / libzip from source" + $zlibProdName = "zlib-1.2.8" + $zlibFilename = "zlib128.zip" + $zlibDownloadSource = "http://zlib.net/zlib128.zip" + $libzipProdName = "libzip-1.1.3" + $libzipFilename = "libzip-1.1.3.tar.gz" + $libzipDownloadSource = "https://nih.at/libzip/libzip-1.1.3.tar.gz" + + $prodSubDir = "zlib-vs15" + $batchFile = "buildZlibVS15.cmd" + + $sourceCodeDir = join-path $targetFolder "src" + $scriptDirectory = join-path $targetFolder "script" + $targetPath = join-path $targetFolder $prodSubDir + $envVar = "ZLIB_PATH" + $envValue = $targetPath + + @( @{ShortName = "ZLIBVS15"; VerifyInfo = "Checking for $prodName in $targetPath"; ActionInfo = "Installing $prodName"; + Verification = @( @{Function = "VerifyDirectory"; Path = "$sourceCodeDir\$zlibProdName" }, + @{Function = "VerifyDirectory"; Path = "$sourceCodeDir\$libzipProdName" }, + @{Function = "VerifyFile"; Path = "$scriptDirectory\$batchFile" } ); + Download = @( @{ Function = "Download"; Source = $zlibDownloadSource; Destination = "$cache\$zlibFilename" }, + @{ Function = "Download"; Source = $libzipDownloadSource; Destination = "$cache\$libzipFilename" } ); + Action = @( @{Function = "ExtractAllFromZip"; zipFileName = "$cache\$zlibFilename"; destination = $sourceCodeDir; zipSubTree =$zlibProdName; destinationFolder =$zlibProdName }, + @{Function = "ExtractAllFromTarGz"; SourceFile = "$cache\$libzipFilename"; TargzFileName = "$libzipFilename"; destination = $sourceCodeDir }, + @{Function = "MakeDirectory"; Path = $scriptDirectory }, + @{Function = "CreateBuildZlibBatch"; FileName = "$scriptDirectory\$batchFile"; zlibSourceDir = (join-path $sourceCodeDir $zlibProdName); libzipSourceDir = (join-path $sourceCodeDir $libzipProdName); TargetDir = $targetPath } ); + } ) +} + + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/PreRequisites.ps1 b/Scripts/devInstall/Windows/helper/PreRequisites.ps1 new file mode 100644 index 000000000..259f48081 --- /dev/null +++ b/Scripts/devInstall/Windows/helper/PreRequisites.ps1 @@ -0,0 +1,95 @@ +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# +function PreReqOperations( + [array] $actionList = @()) +{ + foreach ($item in $actionList) { + foreach ($prereqItem in $item.PreReq) { + PreRequisiteItem $prereqItem + } + } + + Write-Host "Install operations finished" + Write-Host +} + +function PreRequisiteItem( + [hashtable] $item) +{ + $func = $item["Function"] + + $expr = $func +' $item' + + Write-Verbose "Calling Operation: [$func]" + Invoke-Expression $expr +} + +function PrereqInfo2013Up5( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + + Write-Host " + +We require the installation of Visual Studio 2013 Update 5 to continue. +Please follow the information here [https://support.microsoft.com/en-us/kb/3021976]. +" +} + +function PrereqInfoVS15( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + Write-Host " + +Installation of Visual Studio 2015 Update 3 is a pre-requisite before installation +can continue. Please check [https://www.visualstudio.com/vs/] for more details on +Visual Studion 2015 +" +return +} + +function PrereqInfoCuda8( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + Write-Host " + +Installation of NVidia Cuda 8.0 is a pre-requisite before installation +can continue. Please check https://developer.nvidia.com/cuda-downloads] for more +details on Cuda download. Please also remember to set the environment variable +CUDA_PATH_V8_0 to the Cuda8 installation location. +" +return +} + +function PrereqInfoCuDnn51( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + Write-Host " + +Installation of NVidia CuDNN 4.1 for Cuda 8.0 is a pre-requisite before installation +can continue. Please check https://developer.nvidia.com/cudn] for more +details on Cuda download. Please also remember to set the environment variable +CUDNN_PATH to the CudDnn folder in your installation location. +" +return +} + +function StopInstallation( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + throw "Not all pre-requisites installed, installation terminated." +} + + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/helper/Verification.ps1 b/Scripts/devInstall/Windows/helper/Verification.ps1 new file mode 100644 index 000000000..c69905f95 --- /dev/null +++ b/Scripts/devInstall/Windows/helper/Verification.ps1 @@ -0,0 +1,366 @@ +# +# Copyright (c) Microsoft. All rights reserved. +# Licensed under the MIT license. See LICENSE.md file in the project root for full license information. +# +function VerifyOperations( + [array] $verificationList) +{ + Write-Host "Determining Operations to perform. This will take a moment..." + + $result = @() + + foreach ($item in $verificationList) { + $needsInstall = $false + Write-Host $item.VerifyInfo + foreach ($verificationItem in $item.Verification) { + $needsInstall = VerifyItem $verificationItem + if (-not $needsInstall) { + $result += $item + break + } + } + } + return $result +} + +function VerifyItem( + [hashtable] $item) +{ + $func = $item["Function"] + $name = $item["Name"] + + $expr = $func +' $item' + + Write-Verbose "Calling Operation: [$func]: [$name]" + $noInstallRequired = Invoke-Expression $expr + return $noInstallRequired +} + +function VerifyScanPrograms( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $noInstallRequired = $true + + # no actual work is being performed, just the script local datastructure with the list + # of installed programs is being initialized + LoadWinProduct + return $noInstallRequired +} + +function VerifyWinProductExists( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $match = $table["Match"] + $compare = GetTableDefaultString -table $table -entryName "Compare" -defaultValue "" + $matchExact = GetTableDefaultBool -table $table -entryName "MatchExact" -defaultValue $true + $noInstallRequired = $true + + $allProducts = LoadWinProduct + $productList = @($allProducts | Where-Object { $_.Name -match $match } ) + + if ($productList.Count -eq 0) { + $noInstallRequired = $false + } + elseif ($compare.length -gt 0) { + if ($matchExact) { + $productList = @($productList | Where-Object { $_.Name -eq $compare }) + } + else { + $productList = @($productList | Where-Object { $_.Version -ge $compare }) + } + if ($productList.Count -eq 0) { + Write-Verbose "No product found matching the compare requirement`n$productList" + $noInstallRequired = $false + } + } + + Write-Verbose "[$func]: Product [$match] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyWinProductVersion( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $match = $table["Match"] + $version = $table["Version"] + $matchExact = GetTableDefaultBool -table $table -entryName "MatchExact" -defaultValue $false + $noInstallRequired = $true + + if ($matchExact -eq $null) { + $matchExact = $false + } + + $allProducts = LoadWinProduct + $productList = @($allProducts | Where-Object { $_.Name -match $match } ) + + if ($productList.Count -eq 0) { + Write-Verbose "No product found with Name matching [$match]" + $noInstallRequired = $false + } + else { + if ($matchExact) { + $productList = @($productList | Where-Object { $_.Version -eq $version }) + } + else { + $productList = @($productList | Where-Object { $_.Version -ge $version }) + } + if ($productList.Count -eq 0) { + Write-Verbose "No product found matching the version requirement`n$productList" + $noInstallRequired = $false + } + } + Write-Verbose "[$func]: Product [$match] Version {$version] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyInstallationContent( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $path = $table["Path"] + + $noInstallRequired = (join-path $path cntk\cntk.exe | test-path -PathType Leaf) + $noInstallRequired = (join-path $path prerequisites\VS2012\vcredist_x64.exe | test-path -PathType Leaf) -and $noInstallRequired + $noInstallRequired = (join-path $path prerequisites\VS2013\vcredist_x64.exe | test-path -PathType Leaf) -and $noInstallRequired + $noInstallRequired = (join-path $path prerequisites\MSMpiSetup.exe | test-path -PathType Leaf) -and $noInstallRequired + + if ($noInstallRequired) { + Write-Verbose "[$func]: [$path] returned [$noInstallRequired]" + return $noInstallRequired + } + + throw "`nFatal Error: Files from CNTK binary download package are missing!`nThe install script must be run out of the unpacked binary CNTK package, not from a CNTK source clone." +} + +function VerifyDirectory( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $path = $table["Path"] + + $noInstallRequired = (test-path -path $path -PathType Container) + + Write-Verbose "[$func]: [$path] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyRunAlways( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + + $noInstallRequired = $false + Write-Verbose "[$func]: returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyWheelDirectory( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $path = $table["WheelDirectory"] + $forceUpdate = $table["ForceUpdate"] + + $noInstallRequired = $false + + Write-Verbose "[$func]: [$path] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyPathIncludes( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $path = $table["Path"] + + $noInstallRequired = (test-path -path $path -PathType Container) + + Write-Verbose "[$func]: [$path] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyDirectoryContent( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $source = $table["Source"] + $dest = $table["Destination"] + + $noInstallRequired = (test-path -path $source -PathType Container) + + if ($noInstallRequired) { + $noInstallRequired = (test-path -path $dest -PathType Container) + } + if ($noInstallRequired) { + $r = Compare-Object $(Get-ChildItem $source -Recurse) $(Get-ChildItem $dest -Recurse) + if ($r) { + $noInstallRequired = $false + } + } + + Write-Verbose "[$func]: [$source] with [$dest] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyFile( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $path = $table["Path"] + + $noInstallRequired = (test-path -path $path -PathType Leaf) + + Write-Verbose "[$func]: [$path] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyRegistryKey( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $key = $table["Key"] + + $noInstallRequired = (test-path -path $key) + + Write-Verbose "[$func]: [$key] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyRegistryKeyName( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $key = $table["Key"] + $regName = $table["RegName"] + + $noInstallRequired = Test-ItemProperty -Path $key -Name $regName + + Write-Verbose "[$func]: [$key]:[$regname] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyRegistryKeyNameData( + [Parameter(Mandatory = $true)][hashtable] $table) +{ + FunctionIntro $table + $func = $table["Function"] + $key = $table["Key"] + $regName = $table["RegName"] + $regData = $table["RegData"] + $orLater = $table["OrLater"] + + if ($orLater -eq $null) { + $orLater = $false + } + + $noInstallRequired = (test-path -path $key) + if ($noInstallRequired) { + $theKeyObj = get-item $key + $theKeyValue = $theKeyObj.GetValue("$regName") + $noInstallRequired = $false + + if ($theKeyValue -ne $null) { + if ($orLater) { + $noInstallRequired = ($theKeyValue -ge $regData) + } + else { + $noInstallRequired = ($theKeyValue -eq $regData) + } + } + } + + Write-Verbose "[$func]: [$key]:[$regname] == [$regData] returned [$noInstallRequired]" + return $noInstallRequired +} + +function VerifyEnvironmentAndData( + [Parameter(Mandatory = $true)][hashtable] $table +) +{ + FunctionIntro $table + $func = $table["Function"] + $name = $table["EnvVar"] + $content = $table["Content"] + $location = "Machine" + + $envContent = [environment]::GetEnvironmentVariable($name, $location) + + $noInstallRequired = ($envContent -ieq $content) + + Write-Verbose "[$func]: [$name] == [$content] returned [$noInstallRequired]" + return $noInstallRequired +} + +function Test-ItemProperty ( + [string] $Path, + [string] $Name) +{ + if (Test-Path $Path) { + try { + $ItemProperty = Get-ItemProperty -Path $Path -Name $Name -ErrorAction SilentlyContinue + if ( $ItemProperty -ne $null ) { + return $true + } + } + catch { + return $false + } + } + return $false +} + +function ClearScriptVariables +{ + $Script:WinProduct = $Null +} + +function LoadWinProduct +{ + if ($Script:WinProduct -eq $Null) { + # + # $Script:WinProduct = Get-WmiObject Win32_Product + # The above line was the previous solution, but iterating through the registry is much faster + # get-wmiobject does more house-holding, like checking for concistency etc ... + # + $allInstalled = @(Get-childitem "HKCU:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall" -ErrorAction SilentlyContinue) + ` + @(Get-childitem "HKLM:\SOFTWARE\Microsoft\Windows\CurrentVersion\Uninstall" -ErrorAction SilentlyContinue) + ` + @(get-childitem "HKLM:\SOFTWARE\WOW6432Node\Microsoft\Windows\CurrentVersion\Uninstall\" -ErrorAction SilentlyContinue) + + $result = @() + foreach($item in $allInstalled) { + $displayName = $item.GetValue("DisplayName") + if ($displayName.Length -gt 0) { + $entry = New-Object PSObject + $entry | Add-Member -MemberType NoteProperty -Name "Name" -Value $displayName + $entry | Add-Member -MemberType NoteProperty -Name "Version" -Value $($item.GetValue("DisplayVersion")) + + $result += $entry + } + + } + $result = $result | Sort-Object Name,Version -Unique + + $Script:WinProduct = $result + } + return $Script:WinProduct +} + + +# vim:set expandtab shiftwidth=2 tabstop=2: \ No newline at end of file diff --git a/Scripts/devInstall/Windows/readme.md b/Scripts/devInstall/Windows/readme.md new file mode 100644 index 000000000..38d2062bc --- /dev/null +++ b/Scripts/devInstall/Windows/readme.md @@ -0,0 +1,118 @@ +## Developer Setup Windows from a script + +This documents describes the setup of the CNTK developer environment. The script follows overall the recipe described in the public documentation of the CNTK developer setup, available [here](https://github.com/Microsoft/CNTK/wiki/Setup-CNTK-on-Windows). + +>Note: Before startingt the script base installation, please check the CNTK Wiki for more detailed instructions (?here?) + +### The installation script + +The most up-to-date version of the installation script can be found on ``\\cntk-muc00\devinstall``. Copy +The prepared installation script to a directory on your system. In the following we assume you are +going to copy the files in to the directory `c:\local\devinstall`. Start a Windows command shell and enter +``` +c: +cd \ +md local && cd local +md devinstall && cd devinstall +robocopy \\cntk-muc00\devinstall . /MIR +``` +The installation script itself is written for the Powershell environment. By default Powershell doesn't +allow the execution of any scripts. To allow execution of the installation script, +start Powershell from a **standard Windows command shell** by: +``` +start powershell -executionpolicy remotesigned +``` +This will open a Powershell environment. In this enviroment change into the directory you copied +the install script into, here you can directly start the installation script +``` +cd c:\local\devinstall +.\devinstall.ps1 +``` +>Note: If you now receive an errormessage stating that running scripts is disabled on your system, +please make sure you started powershell with the changed executionpolicy. This is enabling +script execution in your created Powershell environment. + +The script will inspect your system and determine the components which are missing to build the +Microsoft Cognitive Toolkit. You will be +notified about the proposed installation steps. At this point you are running in a **demo** mode - +NO changes to your system are being performed. + +### Running the installation script + +If you are satisfied with the proposed changes, you can proceed to the actual installation. +The installation script supports several command line options. ``get-help .\install.ps1`` will +give you a list over the available option. At this point it is recommended to start the installation +by adding the `-Execute` parameter: +``` +.\devinstall.ps1 -execute +``` +The script will download needed components from the web, therefore a connection to the Internet is required. +Downloaded components are stored in the directory ``c:\cntkLocalCache`` and can be removed after complete +installation. During execution of the installation script you might receive requests/warnings from UAC +(User Account Control) depending on your system configuration. Please acknowledge the execution and +installation of the downloaded components. + +> Note: Some components of the installation script (i.e. NVidia CUDA install ) might performan a reboot of your +system. You can just start the installation script again. It will analyze your system again and reuse already +downloaded components. + +### Result of the installation script + +The following changes (if necessary) to your system will be performed by the installation script: + +- Installation of the Visual Studio 2012 and Visual Studio 2013 runtime modules +- Installation of Microsoft MPI +- Installation of the Microsoft MPI SDK +- If not present on your system. Git will be installed + - Location: ``%PROGAMFILES%\Git`` + - Environment: Added to the PATH +- NVidia CUDA 7.5 + - Location: ``%PROGRAMFILES(x86)%\NVIDIA Corporation``, ``%PROGRAMFILES%\NVIDIA GPU Computing Toolkit``, + see NVidia documentation for details + - Environment: ``CUDA_PATH``, ``CUDA_PATH_V7_5`` +- NVidia CuDNN 5.1 for CUDA 7.5 + - Location: ``c:\local\cudnn-7.5-v5.1`` + - Environment: ``CUDNN_PATH`` +- NVidia CUB 1.4.1 + - Location: ``c:\local\cub-1.4.1`` + - Environment: ``CUB_PATH`` +- Boost 1.60 + - Location: ``c:\local\boost_1_60_0`` + - Environment: ``BOOST_INCLUDE_PATH``, ``BOOST_LIB_PATH`` +- A CNTK specific MKL library + - Location: ``c:\local\cntkmkl`` + - Environment: ``CNTK_MKL_PATH`` +- OpenCV 3.1.0 + - Location: ``c:\local\Opencv3.1.0`` + - Environment: ``OPENCV_PATH_V31`` +- Zlib - Precompiled version of Zlib and libzip + - Location: ``c:\local\zlib`` + - Environment: ``ZLIB_PATH`` +- Protobuf 3.1.0 libararies and headers + - Location: ``c:\local\protobuf-3.1.0`` + - Environment: ``PROTOBUF_PATH`` +- SWIG 3.0.10 + - Location: ``c:\local\swigwin-3.0.10`` + - Environment: ``SWIG_PATH`` +- CNTK Git repository clone + - Location: ``c:\repos\CNTK`` +- Anaconda3 - 4.1.1 + - Location: ``c:\local\Anaconda3-4.1.1-Windows-x86_64`` + This is a local Anaconda install, it hasn't been added to the path, and is registered only to the current user + +>Note: This script already installed the compiled Protobuf library, as well as the compiled +zlib and libzip components. It isn't necessary to perform the additional compilation steps (listed in the +public documentation) for these components. + +Once the script finished, you should be ready to build. A couple of points to note: + - Depending on the tools installed, the script might have done changes to your system (especially a CUDA install). + Rebooting you system might be a good choice. + - It is possible to run the script multiple times. Powershell is very specific that it doesn't pick up changes to + environment variables which are done in the script, you have to restart powershell to pick up the latest environment. + - If you are planning on using a GPU, you should install the latest GPU driver from NVidia, and reboot + your system. + + + + +