Merge branch 'master' of https://github.com/Microsoft/cntk into peykash/initializer

CNTK.Common.props

@@ -46,6 +46,7 @@
     <PublicBuild>false</PublicBuild>
 
     <PythonWithDeps>false</PythonWithDeps>
+    <PythonWithDeps Condition=" '$(PYTHON_WITH_DEPS)' != '' ">$(PYTHON_WITH_DEPS)</PythonWithDeps>
     <PythonWithDeps Condition="$(PublicBuild)">true</PythonWithDeps>
 
     <!-- CntkVersionProvidedExternally:

Source/CNTKv2LibraryDll/API/CNTKLibrary.h

@@ -4803,6 +4803,11 @@ namespace CNTK
     ///
     CNTK_API FunctionPtr ReconcileDynamicAxes(const Variable& operand, const Variable& axesAsOperand, const std::wstring& name = L"");
 
+    ///
+    /// Creates an instance of custom proxy op, which is used to relay inputs and parameters to an optimized implementation. E.g. Halide.
+    ///
+    CNTK_API FunctionPtr CustomProxyOp(const std::vector<Variable>& operands, const std::wstring& customOp, const NDShape& outputShape, DataType outputType, const std::wstring& name = L"");
+
     namespace Sequence
     {
         CNTK_API FunctionPtr IsFirst(const Variable& operand, const std::wstring& name = L"");

Source/CNTKv2LibraryDll/CompositeFunction.cpp

@@ -1332,6 +1332,11 @@ namespace CNTK
                     }
                     break;
                 }
+                case PrimitiveOpType::CustomProxyOp:
+                {
+                    ASSIGN_NEW_NODE(CustomProxyOpNode, network->GetDeviceId(), internalNodeName);
+                    break;
+                }
                 default:
                     CNTK::LogicError("Specified op %S not yet supported", PrimitiveOpTypeName(op).c_str());
                     break;

Source/CNTKv2LibraryDll/Function.cpp

@@ -1688,6 +1688,16 @@ namespace CNTK
         return UnaryOp(PrimitiveOpType::ConstantOp, operand, std::move(additionalProperties), name);
     }
 
+    FunctionPtr CustomProxyOp(const std::vector<Variable>& operands, const std::wstring& customOp, const NDShape& outputShape, DataType outputType, const std::wstring& name)
+    {
+        auto attributes = Dictionary();
+        attributes[PrimitiveFunction::AttributeNameCustomOp] = customOp;
+        attributes[PrimitiveFunction::AttributeNameOutputShape] = outputShape;
+        attributes.Add(PrimitiveFunction::AttributeNameNewDataType, static_cast<int>(outputType));
+        auto op = AsComposite(MakeSharedObject<PrimitiveFunction>(PrimitiveOpType::CustomProxyOp, operands, std::move(attributes), name));
+        return op;
+    }
+    
     FunctionPtr EyeLike(const Variable& operand, bool isOutputSparse, const std::wstring& name)
     {
         auto additionalProperties = Dictionary();

Source/CNTKv2LibraryDll/PrimitiveFunction.cpp

@@ -249,6 +249,30 @@ namespace CNTK
             outputs.assign(m_inputs.begin(), m_inputs.end());
         else if (m_op == PrimitiveOpType::NoOp)
             outputs.push_back(OutputVariable(m_inputs[0].Shape(), m_inputs[0].GetDataType(), m_inputs[0].DynamicAxes(), m_inputs[0].NeedsGradient(), Name()));
+        else if (m_op == PrimitiveOpType::CustomProxyOp)
+        {
+            // Set the output data type and shape using attributes.
+            DataType outputDataType = DataType::Unknown;
+            if (m_attributes.Contains(PrimitiveFunction::AttributeNameNewDataType))
+            {
+                outputDataType = static_cast<DataType>(m_attributes[PrimitiveFunction::AttributeNameNewDataType].Value<int>());
+            }
+            else
+            {
+                InvalidArgument("Output type must be specified for CustomProxyOp.");
+            }
+            NDShape outputShape = NDShape::Unknown();
+            if (m_attributes.Contains(PrimitiveFunction::AttributeNameOutputShape))
+            {
+                outputShape = m_attributes[PrimitiveFunction::AttributeNameOutputShape].Value<NDShape>();
+            }
+            else
+            {
+                InvalidArgument("Output shape must be specified for CustomProxyOp.");
+            }
+            
+            outputs.push_back(OutputVariable(outputShape, outputDataType, m_inputs[0].DynamicAxes(), false, Name().empty() ? L"" : Name()));
+        }
         else
         {
             DataType outputDataType = GetOutputDataType(m_op, m_inputs, true);

Source/CNTKv2LibraryDll/PrimitiveFunction.h

@@ -118,6 +118,7 @@ namespace CNTK
         {PrimitiveOpType::Squeeze, L"Squeeze"},
         {PrimitiveOpType::Cast, L"Cast" },
         { PrimitiveOpType::EyeLikeOp, L"EyeLikeOp" },
+        { PrimitiveOpType::CustomProxyOp, L"CustomProxyOp" },
     };
 
     inline const std::wstring& PrimitiveOpTypeName(PrimitiveOpType opType)
@@ -317,6 +318,7 @@ namespace CNTK
         static const std::wstring AttributeNameUseStatsAcrossChannels;
         static const std::wstring AttributeNameDoVarianceScaling;
         static const std::wstring AttributeNameGroups;
+        static const std::wstring AttributeNameCustomOp;
 
         static const size_t convolutionOpDefaultValueForGroups = 1;
 

Source/CNTKv2LibraryDll/PrimitiveFunctionAttributes.h

@@ -119,4 +119,5 @@ namespace CNTK
     /*static*/ const std::wstring PrimitiveFunction::AttributeNameUseStatsAcrossChannels = L"useStatsAcrossChannels";
     /*static*/ const std::wstring PrimitiveFunction::AttributeNameDoVarianceScaling = L"doVarianceScaling";
     /*static*/ const std::wstring PrimitiveFunction::AttributeNameGroups = L"groups";
+    /*static*/ const std::wstring PrimitiveFunction::AttributeNameCustomOp = L"customOp";
 }

Source/CNTKv2LibraryDll/PrimitiveOpType.h

@@ -102,6 +102,8 @@ namespace CNTK
         LatticeSequenceWithSoftmax = 90,
         Cast = 91,
         EyeLikeOp = 92,
+        CustomProxyOp = 93,
+
         // New op types should only be appended to the end of this list 
         UnknownOP
         // and UnknownOP should always be last.

Source/CNTKv2LibraryDll/Variable.cpp

@@ -145,6 +145,11 @@ namespace CNTK
                     m_dataFields->m_value = CreateValueFromParameterInitializer<half>(Shape(), *m_dataFields->m_valueInitializer, *m_dataFields->m_valueInitializationDevice);
                     break;
                 }
+                case DataType::Int8:
+                {
+                    m_dataFields->m_value = CreateValueFromParameterInitializer<char>(Shape(), *m_dataFields->m_valueInitializer, *m_dataFields->m_valueInitializationDevice);
+                    break;
+                }
                 default:
                     LogicError("Variable '%S' Value(): Unsupported DataType %s", AsString().c_str(), DataTypeName(GetDataType()));
                     break;
@@ -507,7 +512,8 @@ namespace CNTK
         if (dataType != DataType::Unknown &&
             dataType != DataType::Float &&
             dataType != DataType::Double &&
-            dataType != DataType::Float16)
+            dataType != DataType::Float16 &&
+            dataType != DataType::Int8)
         {
             LogicError("Unexpected variable datatype '%ls':'%u' (%s).", 
                        dataTypeKey.c_str(), 

Source/ComputationNetworkLib/InputAndParamNodes.cpp

@@ -355,6 +355,19 @@ void LearnableParameter<char>::InitBilinear(Matrix<char>& valueMatrix, const Ten
     RuntimeError("Unsupported template argument(char) in InitBilinear");
 }
 
+template <>
+std::tuple<size_t, size_t, char> LearnableParameter<char>::InitRandom(Matrix<char>& valueMatrix,
+    const TensorShape& sampleShape,
+    const wstring& type,
+    const unsigned long randomSeed,
+    const char initValueScale,
+    const size_t initFilterRank,
+    const int initOutputRank,
+    const bool initOnCPUOnly,
+    DEVICEID_TYPE deviceId)
+{
+    RuntimeError("Unsupported template argument(char) in InitRandom");
+}
 
 // initialize by reading a matrix from a text file
 template <class ElemType>

Source/ComputationNetworkLib/SpecialPurposeNodes.h

@@ -1432,4 +1432,41 @@ private:
 template class OutputMultiplexerNode<float>;
 template class OutputMultiplexerNode<double>;
 
+// -----------------------------------------------------------------------
+// CustomProxyOpNode is a placeholder node for a quantized operations.
+// It enables saving a model with its parameters so that they can be loaded
+// from the optimized implementation (Halide) for execution.
+// -----------------------------------------------------------------------
+
+template <class ElemType>
+class CustomProxyOpNode : public ComputationNode<ElemType> /* Not deriving from NumInputs, public NumInputs<4>*/
+{
+    typedef ComputationNode<ElemType> Base; UsingComputationNodeMembersBoilerplate;
+    static const std::wstring TypeName() { return L"CustomProxyOpNode"; }
+
+public:
+    CustomProxyOpNode(DEVICEID_TYPE deviceId, const wstring& name)
+        : Base(deviceId, name)
+    {
+    }
+
+    CustomProxyOpNode(const ScriptableObjects::IConfigRecordPtr configp)
+        : CustomProxyOpNode(configp->Get(L"deviceId"), L"<placeholder>")
+    {
+        AttachInputsFromConfig(configp);
+    }
+
+    virtual void /*ComputationNode::*/ ForwardProp(const FrameRange& fr) override
+    {
+        NOT_IMPLEMENTED
+    }
+
+    virtual void /*ComputationNode::*/ BackpropTo(const size_t inputIndex, const FrameRange& fr) override
+    {
+        NOT_IMPLEMENTED
+    }
+};
+
+template class CustomProxyOpNode<float>;
+
 } } }

bindings/python/cntk/internal/tests/types_test.py

@@ -48,3 +48,22 @@ def test_saving_and_loading_int8_ndarray_as_attribute(tmpdir):
     assert(int8_data.shape == (16,4))
 
     assert (np.array_equal(int8_data, data))
+
+def test_custom_op_with_int8_params(tmpdir):
+    model_file = str(tmpdir/'test_model_params.bin')
+    delete_if_file_exists(model_file)
+
+    W1 = C.Parameter((1, 42), dtype=np.int8)
+    W1.value = np.arange(42).reshape(1, 42)
+    W2 = C.Parameter((1, 42), dtype=np.int8)
+    W3 = C.Parameter((1, 42), dtype=np.float)
+    X = C.input_variable((1, 42), dtype=np.float)
+
+    # custom_op, output_shape, output_data_type, *operands, **kw_name
+    z = C.custom_proxy_op("times", (21, 2), np.int8, X, W1, W2, W3, name ="custom_proxy")
+    z.save(model_file)
+
+    newz = C.load_model(model_file)
+    assert(newz.parameters[0].shape == (1, 42))
+    assert(newz.output.shape == (21, 2))
+    assert (np.array_equal(W1.value, newz.parameters[0].value))

bindings/python/cntk/ops/__init__.py

@@ -3989,4 +3989,32 @@ def mean_variance_normalization(operand, epsilon=0.00001, use_stats_across_chann
     if epsilon < 0:
         raise ValueError('epsilon must be non-negative.')
     operand = sanitize_input(operand, get_data_type(operand))  
-    return mean_variance_normalization(operand, epsilon, use_stats_across_channels, do_variance_scaling, name)
+    return mean_variance_normalization(operand, epsilon, use_stats_across_channels, do_variance_scaling, name)
+
+@typemap
+def custom_proxy_op(custom_op, output_shape, output_data_type, *operands, **kw_name):
+    '''
+    A proxy node that helps saving a model with different number of operands. 
+
+    Example:
+
+    Args:
+
+    Returns:
+        :class:`~cntk.ops.functions.Function`
+    '''
+    from cntk.cntk_py import custom_proxy_op
+    if len(operands) == 1 and isinstance(operands[0], (tuple, list)):
+        operands = operands[0]
+    if isinstance(operands, tuple):
+        operands = list(operands)
+    operands_unfold = []
+    for o in operands:
+        if hasattr(o, 'outputs') and len(o.outputs) > 1:
+            operands_unfold += o.outputs
+        else:
+            operands_unfold += [o]
+
+    name = (lambda name='': (name))(**kw_name) # Python 2.7 does not allow (*inputs, name='')
+    output_dtype = sanitize_dtype_cntk(output_data_type)
+    return custom_proxy_op(operands_unfold, custom_op, output_shape, output_dtype, name)

configure

@@ -345,7 +345,15 @@ enable_python=$(default_use_python)
 
 function default_python_with_deps ()
 {
-        echo $default_build_public
+    if [[ $default_build_public == "yes" ]]; then
+        echo yes
+    else
+        if [[ -z "$PYTHON_WITH_DEPS" ]]; then
+            echo no
+        else
+            echo $PYTHON_WITH_DEPS
+        fi
+    fi
 }
 python_with_deps=$(default_python_with_deps)