ConvTranspose asymmetric padding

* Temporarily reverse the extra padding location in case of
SAME_UPPER vs SAME_LOWER for convTranspose to match with onnxruntime.
* In case of importing asymmetric padding convTranspose, use
symmetric pads by alter the output_shape and pads, and attach a slice
node afterwards to enable cudnn.
* Fix a bug in slice/squeeze attribute axes export.

Source/CNTKv2LibraryDll/proto/onnx/CNTKToONNX.cpp

@@ -535,7 +535,6 @@ private:
     static onnx::TypeProto ToTypeProto(const NDShape& shape, bool hasBatchAxis = false, bool hasSequenceAxis = false, bool doReverseShape = true);
     static onnx::TypeProto ToTypeProto(const std::vector<bool>& shape);
     static onnx::TypeProto ToTypeProto(const std::vector<int64_t>& shape, bool doReverseVec = true);
-    static onnx::TypeProto ToTypeProto(const std::vector<Axis>& axes);
 
     //
     // Convert TypeProto, NDShape and various std::vector types to std::vector
@@ -544,7 +543,6 @@ private:
     static std::vector<int64_t> ToINTS(const NDShape& shape, bool hasBatchAxis = false);
     static std::vector<int64_t> ToINTS(const std::vector<bool>& shape);
     static std::vector<int64_t> ToINTS(const std::vector<int>& shape, bool doReverseVec = true);
-    static std::vector<int64_t> ToINTS(const std::vector<Axis>& axes);
 
     static std::vector<float> INTSToVecFloat(const std::vector<int64_t> &ints);
     static std::vector<int64_t> ConvertPermutationCNTKToONNX(const std::vector<Axis> &axes, bool hasBatchAxis);
@@ -1688,23 +1686,6 @@ onnx::TypeProto CNTKToONNXHelper::ToTypeProto(const std::vector<int64_t>& shape,
     return newShape;
 }
 
-onnx::TypeProto CNTKToONNXHelper::ToTypeProto(const std::vector<Axis>& axes)
-{
-    std::vector<int> axesValue;
-    for (auto axis : axes)
-    {
-        axesValue.push_back(ToIndex(axis));
-    }
-    std::sort(axesValue.begin(), axesValue.end());
-
-    onnx::TypeProto newShape = MakeTypeProtoWithShape();
-
-    for (auto dimension : axesValue)
-        newShape.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(dimension);
-
-    return newShape;
-}
-
 // this method is to undo an idempotent convertion in sanitize_permutation:
 // Find the permutation such that when it is applied to the reverse
 // of an input gives the reverse of perm applied to the input
@@ -1768,11 +1749,6 @@ std::vector<int64_t> CNTKToONNXHelper::ToINTS(const std::vector<int>& shape,
     return ToINTS(ToTypeProto(Cast<int, int64_t>(shape), doReverseVec));
 }
 
-std::vector<int64_t> CNTKToONNXHelper::ToINTS(const std::vector<Axis>& axes)
-{
-    return ToINTS(ToTypeProto(axes));
-}
-
 bool IsUnSupportedLayerNormalization(const FunctionPtr src)
 {
     std::string cntkOpName = ToLegacyString(ToUTF8(src->OpName()));
@@ -5542,7 +5518,7 @@ void CNTKToONNXHelper::CopyAttributes(const FunctionPtr& src, onnxruntime::Node*
             if (src->Attributes().Contains(L"axisVec"))
             {
                 std::vector<Axis> sliceAxes = AsVector<Axis>(src->Attributes()[L"axisVec"].Value<std::vector<DictionaryValue>>());
-                node->AddAttribute(attributesMap[L"axes"], ToINTS(sliceAxes));
+                node->AddAttribute(attributesMap[L"axes"], ConvertAxesToOnnx(sliceAxes, src->Inputs()[0]));
 
                 beginIndex = AsVector<int>(src->Attributes()[L"beginIndexVec"].Value<std::vector<DictionaryValue>>());
                 endIndex = AsVector<int>(src->Attributes()[L"endIndexVec"].Value<std::vector<DictionaryValue>>());
@@ -5661,7 +5637,10 @@ void CNTKToONNXHelper::CopyAttributes(const FunctionPtr& src, onnxruntime::Node*
             {
                 axes.push_back((Axis)(src->Attributes()[L"axis"].Value<Axis>()));
             }
-            node->AddAttribute("axes", ToINTS(axes));
+            if (axes.size() > 0)
+            {
+                node->AddAttribute("axes", ConvertAxesToOnnx(axes, src->Inputs()[0]));
+            }
         }
         else if (src->OpName() == L"Gather")
         {

Source/CNTKv2LibraryDll/proto/onnx/ONNXToCNTK.cpp

@@ -1431,6 +1431,8 @@ ConvAutoPadType ONNXToCNTKHelper::ConvertStrToConvAutoPadType(const string &str)
         return ConvAutoPadType::SAME_UPPER;
     else if (str == "SAME_LOWER" || str == "same_lower")
         return ConvAutoPadType::SAME_LOWER;
+    else if (str == "NOTSET" || str == "notset")
+        return ConvAutoPadType::NOTSET;
     else
         LogicError("Unknown value for %s attribute: %s", "auto_pad", str.c_str());
 }
@@ -3322,7 +3324,10 @@ FunctionPtr ONNXToCNTKHelper::CreateCNTKConvTransposeNode(const Node *node, cons
         outputShape = GetNamedAttributeAsShape(node, "output_shape", /*hasBatchAxis=*/false);
         if ((outputShape.Rank() != numSpatialDim) && (outputShape.Rank() != numSpatialDim + 2))
             LogicError("ConvTranspose node's output shape attribute is of unexpected length. It should be either equal to input shape length, or input shape length - 2");
-        padsPair = CalcPaddingFromOutputShape(inputShape, kernelShape, strides, outputShape, outputPadding, /*isSameUpper=*/false);
+        // NOTE: The following is for ONNX V1.3 opset8. It is subject to change in future versions.
+        // For convTranspose, extra pad location is flipped compared to conv/pooling. Thus the flag 'isSameUpper' is flipped to 'notSameUpper'.
+        const bool notSameUpper = ConvertStrToConvAutoPadType(GetNamedAttributeAsString(node, "auto_pad", "SAME_UPPER")) != ConvAutoPadType::SAME_UPPER;
+        padsPair = CalcPaddingFromOutputShape(inputShape, kernelShape, strides, outputShape, outputPadding, notSameUpper);
     }
     else if (USE_PADS)
     {
@@ -3345,9 +3350,10 @@ FunctionPtr ONNXToCNTKHelper::CreateCNTKConvTransposeNode(const Node *node, cons
         case ConvAutoPadType::SAME_UPPER:
         case ConvAutoPadType::SAME_LOWER:
         {
-            const bool isSameUpper = auto_pad == ConvAutoPadType::SAME_UPPER;
+            const bool notSameUpper = auto_pad != ConvAutoPadType::SAME_UPPER;
             auto outputPadding = (HasNamedAttribute(node, "output_padding")) ? GetNamedAttributeAsInt64Vec(node, "output_padding") : std::vector<int64_t>(numSpatialDim, 0);
-            padsPair = CalcPaddingFromOutputShape(inputShape, kernelShape, strides, outputShape, outputPadding, isSameUpper);
+            // For convTranspose, extra pad location is flipped compared to conv/pooling. Thus the flag 'isSameUpper' is flipped to 'notSameUpper'.
+            padsPair = CalcPaddingFromOutputShape(inputShape, kernelShape, strides, outputShape, outputPadding, notSameUpper);
             break;
         }
         case ConvAutoPadType::VALID:
@@ -3366,6 +3372,8 @@ FunctionPtr ONNXToCNTKHelper::CreateCNTKConvTransposeNode(const Node *node, cons
 
     padsPair.first.push_back(0);
     padsPair.second.push_back(0);
+    if (padsPair.first.size() != padsPair.second.size())
+        LogicError("ConvTranspose: producing uneven lower/upper pads rank: lower(%zu), upper(%zu). ", padsPair.first.size(), padsPair.second.size());
 
     // CNTK only accepts outputShape in the format of
     //  case 1: [0]. Empty shape which tells CNTK to infer output shape from other inputs. 
@@ -3396,18 +3404,59 @@ FunctionPtr ONNXToCNTKHelper::CreateCNTKConvTransposeNode(const Node *node, cons
         LogicError("ConvTranspose: unable to produce CNTK compatible output shape from given ONNX node. ");
     }
 
+    // cuDNN couldn't support cases of asymmetric pad values.
+    // Solution: increase the outputShape with size of extra pads, and add a slice node after convTranspose to remove the padded values.
+    std::vector<int> extraUpperPads(outputShape.Rank(), 0);
+    if (extraUpperPads.size() > 1)
+    {
+        assert(padsPair.first.size() == padsPair.second.size());
+        if (padsPair.first.size() != outputShape.Rank())
+            LogicError("ConvTranspose: producing uneven pads rank and outputShape rank: pads(%zu), outputShape(%zu). ", padsPair.first.size(), outputShape.Rank());
+        for (int idx = 0; idx < outputShape.Rank() - 1; ++idx)
+        {
+            if (padsPair.second[idx] != padsPair.first[idx])
+            {
+                extraUpperPads[idx] = padsPair.second[idx] - padsPair.first[idx];
+                if (extraUpperPads[idx] > 0)
+                    padsPair.second[idx] -= extraUpperPads[idx];
+                else
+                    padsPair.first[idx] += extraUpperPads[idx];
+                outputShape[idx] += abs(extraUpperPads[idx]);
+            }
+        }
+    }
+
     FunctionPtr cntkConvFunction = CreateCNTKConvTransposeNode(inputOperand, convolutionMap,
         strides, sharing, padsPair.first, padsPair.second, outputShape,
         dilation, reductionRank, maxTempMemSizeInSamples, node->Name());
 
+    if (std::any_of(extraUpperPads.begin(), extraUpperPads.end(), [](int i) { return i != 0; }))
+    {
+        // Add slice node to remove output values that are considered padded.
+        std::vector<Axis> axes;
+        std::vector<int> beginIndices;
+        std::vector<int> endIndices;
+        for (int idx = 0; idx < extraUpperPads.size() - 1; ++idx)
+        {
+            if (extraUpperPads[idx] != 0)
+            {
+                int extraUpperPad = extraUpperPads[idx];
+                axes.push_back(Axis(idx));
+                beginIndices.push_back(extraUpperPad > 0 ? 0 : -extraUpperPad);
+                endIndices.push_back(extraUpperPad > 0 ? outputShape[idx] - extraUpperPad : outputShape[idx]);
+            }
+        }
+        cntkConvFunction = Slice(cntkConvFunction, axes, beginIndices, endIndices);
+    }
+
     // If Bias is specified in the ONNX node.
     if (inputs.size() == 3)
     {
         NDShape shape({ 1, 1, inputs[2].Shape()[0] });
-        return Plus(cntkConvFunction, Reshape(inputs[2], shape));
+        cntkConvFunction = Plus(cntkConvFunction, Reshape(inputs[2], shape));
     }
-    else
-        return cntkConvFunction;
+
+    return cntkConvFunction;
 }
 
 FunctionPtr ONNXToCNTKHelper::CreateCNTKConvTransposeNode(const Variable& inputOperand, const Variable& convolutionMap, const NDShape& strides,

Source/CNTKv2LibraryDll/proto/onnx/ONNXToCNTK.h

@@ -22,6 +22,7 @@ namespace CNTK
             VALID = 0,
             SAME_UPPER = 1,
             SAME_LOWER = 2,
+            NOTSET = 3,
         };
     }
 

Source/Math/CuDnnConvolutionEngine.cu

@@ -760,9 +760,9 @@ bool CuDnnConvolutionEngineFactory<ElemType>::IsSupported(DEVICEID_TYPE deviceId
             auto lowerPad = geometry->GetLowerPad(i);
             auto upperPad = geometry->GetUpperPad(i);
             auto stride = geometry->GetStride(i);
-            if (kernel[i] % 2 == 0 && lowerPad < upperPad && stride < input[i])
+            if (lowerPad < upperPad && stride < input[i])
             {
-                fprintf(stderr, "WARNING: Detected asymmetric padding issue with even kernel size and lowerPad (%d) < higherPad (%d) (i=%d), cuDNN will not be able to produce correct result. Switch to reference engine (VERY SLOW). \n", lowerPad, upperPad, i);
+                fprintf(stderr, "WARNING: Detected asymmetric padding issue with lowerPad (%d) < higherPad (%d) (i=%d), cuDNN will not be able to produce correct result. Switch to reference engine (VERY SLOW). \n", lowerPad, upperPad, i);
                 retVal = false;
                 break;
             }

bindings/python/cntk/tests/onnx_op_test.py

@@ -1739,9 +1739,14 @@ def test_Slice(tmpdir, dtype):
         model = C.slice(x1, 0, 1, 2)
         verify_one_input(model, data, tmpdir, 'Slice_1')
 
-        model = C.slice(x1, [0,1], [1,0], [2,1]);
+        model = C.slice(x1, [0,1], [1,0], [2,1])
         verify_one_input(model, data, tmpdir, 'Slice2_1')
 
+        data = np.asarray([[[1,1,1,1],[2,2,2,2],[3,3,2,2]], [[4,4,5,5], [5,5,6,6], [6,6,7,7]]],dtype=dtype)
+        x1 = C.input_variable((2,3,4))
+        model = C.slice(x1, [1,2], [1,0],[2,1])
+        verify_one_input(model, data, tmpdir, 'Slice3_1')
+
 #Sequence.Slice 
 @pytest.mark.parametrize("beginIndex, endIndex", (  
     (-2, -1), (0, -1), (1, -1), (-1, 0), (1, 0), (-4, 2), (0, 1), (1, 2)))
@@ -1827,11 +1832,19 @@ def test_Softsign(tmpdir, dtype):
         verify_no_input(model, tmpdir, 'Softsign_0')
 
 #Squeeze
-#def test_Squeeze(tmpdir):
-#    x0 = np.arange(12).reshape((2, 2, 1, 3)).astype('f')
-#    x = C.input_variable((2, 1, 3))
-#    model = C.squeeze(x)
-#    verify_one_input(model, x0, tmpdir, 'Squeeze_0')
+def test_Squeeze(tmpdir):
+    pytest.skip('TODO: need to bump ONNX CI version. ')
+    x0 = np.arange(6).reshape((1, 2, 1, 3)).astype('f')
+    x = C.input_variable((2, 1, 3))
+    model = C.squeeze(x, [1])
+    verify_one_input(model, x0, tmpdir, 'Squeeze_0')
+
+def test_Squeeze_without_axes(tmpdir):
+    pytest.skip('ONNX should update attribute axes to be optional.')
+    x0 = np.arange(6).reshape((1, 2, 1, 3)).astype('f')
+    x = C.input_variable((2, 1, 3))
+    model = C.squeeze(x)
+    verify_one_input(model, x0, tmpdir, 'Squeeze_without_axes_0')
 
 #Sum
 @pytest.mark.parametrize("dtype", DType_Config)