created ConvNet.cntk

Examples/Image/Miscellaneous/CIFAR-10/ConvNet.cntk

@@ -0,0 +1,157 @@
+# Simple CIFAR-10 convnet, without and with BatchNormalization.
+
+command = TrainConvNet:Eval
+#command = TrainConvNetWithBN:Eval
+
+makeMode = false ; traceLevel = 1 ; deviceId = 0
+
+RootDir = "." ; DataDir  = "$RootDir$" ; ModelDir = "$RootDir$/Output/Models"
+
+modelPath = "$ModelDir$/01_Convolution"
+
+# Training without BN
+TrainConvNet = [
+    action = "train"
+
+    BrainScriptNetworkBuilder = [
+        imageShape = 32:32:3
+        labelDim = 10
+
+        Subtract128 (x) = x - Constant (128)
+
+        model = Sequential (
+            Subtract128 :
+            ConvolutionalLayer {32, (5:5), activation = ReLU, init = "gaussian", initValueScale = 0.0043} :
+              MaxPoolingLayer {(3:3), stride = (2:2)} :
+            ConvolutionalLayer {32, (5:5), activation = ReLU, init = "gaussian", initValueScale = 1.414} :
+              MaxPoolingLayer {(3:3), stride = (2:2)} :
+            ConvolutionalLayer {64, (5:5), activation = ReLU, init = "gaussian", initValueScale = 1.414} :
+              MaxPoolingLayer {(3:3), stride = (2:2)} :
+            DenseLayer {64, activation = ReLU, init = "gaussian", initValueScale = 12} :
+              Dropout :
+            LinearLayer {labelDim, init = "gaussian", initValueScale = 1.5}
+        )
+
+        # inputs
+        features = Input {imageShape}
+        labels   = Input {labelDim}
+
+        # apply model to features
+        z = model (features)
+
+        # connect to system
+        ce       = CrossEntropyWithSoftmax (labels, z)
+        errs     = ErrorPrediction         (labels, z)
+        top5Errs = ErrorPrediction         (labels, z, topN=5)  # only used in Eval action
+
+        featureNodes    = (features)
+        labelNodes      = (labels)
+        criterionNodes  = (ce)
+        evaluationNodes = (errs)  # top5Errs only used in Eval
+        outputNodes     = (z)
+    ]
+
+    SGD = [
+        epochSize = 49984 ; minibatchSize = 64
+
+        learningRatesPerSample = 0.00015625*10:0.000046875*10:0.000015625
+        momentumAsTimeConstant = 600*20:6400
+        maxEpochs = 30
+        L2RegWeight = 0.03
+        dropoutRate = 0*5:0.5
+
+        firstMBsToShowResult = 10 ; numMBsToShowResult = 500
+    ]
+
+    reader = [
+        readerType = "CNTKTextFormatReader"
+        file = "$DataDir$/Train_cntk_text.txt"
+        input = [
+            features = [ dim = 3072 ; format = "dense" ]
+            labels   = [ dim = 10 ;   format = "dense" ]
+        ]
+    ]
+]
+
+# Training with BN
+# Not working well on this small set. Overtrains.
+TrainConvNetWithBN = [
+    action = "train"
+
+    BrainScriptNetworkBuilder = [
+        imageShape = 32:32:3
+        labelDim = 10
+
+        Subtract128 (x) = x - Constant (128)
+
+        model = Sequential (
+            Subtract128 :
+            ConvolutionalLayer {32, (5:5), bias = false, init = "gaussian", initValueScale = 0.0043} :
+              BatchNormalizationLayer {spatialRank = 2, normalizationTimeConstant = 4096} : ReLU :
+                MaxPoolingLayer {(3:3), stride = (2:2)} :
+            ConvolutionalLayer {32, (5:5), bias = false, init = "gaussian", initValueScale = 1.414} :
+              BatchNormalizationLayer {spatialRank = 2, normalizationTimeConstant = 4096} : ReLU :
+                MaxPoolingLayer {(3:3), stride = (2:2)} :
+            ConvolutionalLayer {64, (5:5), bias = false, init = "gaussian", initValueScale = 1.414} :
+              BatchNormalizationLayer {spatialRank = 2, normalizationTimeConstant = 4096} : ReLU :
+                MaxPoolingLayer {(3:3), stride = (2:2)} :
+            LinearLayer {64, bias = false, init = "gaussian", initValueScale = 12} :
+              BatchNormalizationLayer {normalizationTimeConstant = 4096} : ReLU :
+            LinearLayer {labelDim, init = "gaussian", initValueScale = 1.5}
+        )
+
+        # inputs
+        features = Input {imageShape}
+        labels   = Input {labelDim}
+
+        # apply model to features
+        z = model (features)
+
+        # connect to system
+        ce       = CrossEntropyWithSoftmax (labels, z)
+        errs     = ErrorPrediction         (labels, z)
+        top5Errs = ErrorPrediction         (labels, z, topN=5)
+
+        featureNodes    = (features)
+        labelNodes      = (labels)
+        criterionNodes  = (ce)
+        evaluationNodes = (errs)
+        outputNodes     = (z)
+    ]
+
+    SGD = [
+        epochSize = 49984 ; minibatchSize = 64
+
+        learningRatesPerSample = 0.00046875*7:0.00015625
+        momentumAsTimeConstant = 0
+        maxEpochs = 10
+        L2RegWeight = 0
+        dropoutRate = 0
+
+        firstMBsToShowResult = 10 ; numMBsToShowResult = 500
+    ]
+
+    reader = [
+        readerType = "CNTKTextFormatReader"
+        file = "$DataDir$/Train_cntk_text.txt"
+        input = [
+            features = [ dim = 3072 ; format = "dense" ]
+            labels   = [ dim = 10 ;   format = "dense" ]
+        ]
+    ]
+]
+
+# Eval action
+Eval = [
+    action = "eval"
+    minibatchSize = 16
+    evalNodeNames = errs:top5Errs  # also test top-5 error rate
+    reader = [
+        readerType = "CNTKTextFormatReader"
+        file = "$DataDir$/Test_cntk_text.txt"
+        input = [
+            features = [ dim = 3072 ; format = "dense" ]
+            labels   = [ dim = 10 ;   format = "dense" ]
+        ]
+    ]
+]