Merge branch 'master' into ebarsoum/globalpooling

For checkin and Jenkins.
2016-11-10 09:49:09 -08:00 · 2016-11-10 09:49:09 -08:00 · 2cd3466b9c
--- a/CNTK.sln
+++ b/CNTK.sln
@ -2263,8 +2263,8 @@ Global
 		{5D29C76D-648A-456F-920D-48230F2FB3C8}.Release_NoOpt|Any CPU.ActiveCfg = Release_CpuOnly|x64
 		{5D29C76D-648A-456F-920D-48230F2FB3C8}.Release_NoOpt|Mixed Platforms.ActiveCfg = Release_CpuOnly|x64
 		{5D29C76D-648A-456F-920D-48230F2FB3C8}.Release_NoOpt|Mixed Platforms.Build.0 = Release_CpuOnly|x64
-		{5D29C76D-648A-456F-920D-48230F2FB3C8}.Release_NoOpt|x64.ActiveCfg = Release_CpuOnly|x64
-		{5D29C76D-648A-456F-920D-48230F2FB3C8}.Release_NoOpt|x64.Build.0 = Release_CpuOnly|x64
+		{5D29C76D-648A-456F-920D-48230F2FB3C8}.Release_NoOpt|x64.ActiveCfg = Release_NoOpt|x64
+		{5D29C76D-648A-456F-920D-48230F2FB3C8}.Release_NoOpt|x64.Build.0 = Release_NoOpt|x64
 		{5D29C76D-648A-456F-920D-48230F2FB3C8}.Release|Any CPU.ActiveCfg = Release|x64
 		{5D29C76D-648A-456F-920D-48230F2FB3C8}.Release|Mixed Platforms.ActiveCfg = Release|x64
 		{5D29C76D-648A-456F-920D-48230F2FB3C8}.Release|Mixed Platforms.Build.0 = Release|x64
--- a/Scripts/linux/install-cntk.sh
+++ b/Scripts/linux/install-cntk.sh
@ -10,7 +10,7 @@
 # TODO cut down on logging
 set -x -e -o pipefail

-REPO_TAG=v2.0.beta2.0
+REPO_TAG=v2.0.beta3.0

 while [ $# -gt 0 ]; do
  case "$1" in
@ -41,7 +41,7 @@ CNTK_DEP_LIB_PATH="$PWD/cntk/dependencies/lib"
 CNTK_EXAMPLES_PATH="$PWD/Examples"
 CNTK_BINARY="$CNTK_BIN_PATH/cntk"
 CNTK_PY34_ENV_FILE="$SCRIPT_DIR/conda-linux-cntk-py34-environment.yml"
-CNTK_WHEEL_PATH="cntk/python/cntk-2.0.beta2.0-cp34-cp34m-linux_x86_64.whl"
+CNTK_WHEEL_PATH="cntk/python/cntk-2.0.beta3.0-cp34-cp34m-linux_x86_64.whl"
 test -d "$CNTK_BIN_PATH" && test -d "$CNTK_LIB_PATH" && test -d "$CNTK_DEP_LIB_PATH" && 
 test -d "$CNTK_EXAMPLES_PATH" && test -x "$CNTK_BINARY" &&
 test -f "$CNTK_PY34_ENV_FILE" && test -f "$CNTK_WHEEL_PATH" || {
--- a/Scripts/windows/install.ps1
+++ b/Scripts/windows/install.ps1
@ -60,7 +60,7 @@
 Param(
    [parameter(Mandatory=$false)] [string] $AnacondaBasePath = "C:\local\Anaconda3-4.1.1-Windows-x86_64",
    [parameter(Mandatory=$false)] [switch] $Execute,
-    [parameter(Mandatory=$false)] [string] $RepoTag="v2.0.beta2.0",
+    [parameter(Mandatory=$false)] [string] $RepoTag="v2.0.beta3.0",
    [parameter(Mandatory=$false)] [string] $RepoLocation="c:\repos\CNTK"
 )

--- a/Source/CNTK/CNTK.cpp
+++ b/Source/CNTK/CNTK.cpp
@ -642,7 +642,7 @@ int wmainWithBS(int argc, wchar_t* argv[]) // called from wmain which is a wrapp

 static void PrintBanner(int argc, wchar_t* argv[], const string& timestamp)
 {
-    fprintf(stderr, "CNTK 2.0.beta2.0+ (");
+    fprintf(stderr, "CNTK 2.0.beta3.0+ (");
 #ifdef _GIT_EXIST
    fprintf(stderr, "%s %.6s, ", _BUILDBRANCH_, _BUILDSHA1_);
 #endif
--- a/Source/CNTKv2LibraryDll/API/CNTKLibrary.h
+++ b/Source/CNTKv2LibraryDll/API/CNTKLibrary.h
@ -2699,6 +2699,16 @@ namespace CNTK
        return TransposeTimes(leftOperand, rightOperand, /*outputRank =*/ 1, name);
    }

+    ///
+    /// Create an instance of the CNTK built-in operation to compute binary cross-entropy for specified input operands.
+    ///
+    CNTK_API FunctionPtr BinaryCrossEntropy(const Variable& prediction, const Variable& targets, const std::wstring& name = L"");
+
+    ///
+    /// Create an instance of the CNTK built-in operation to compute weighted binary cross-entropy for specified input operands.
+    ///
+    CNTK_API FunctionPtr WeightedBinaryCrossEntropy(const Variable& prediction, const Variable& targets, const Variable& weights, const std::wstring& name = L"");
+
    ///
    /// Create an instance of the CNTK built-in operation to compute squared-error for specified input operands.
    ///
--- a/Source/CNTKv2LibraryDll/BackCompat.cpp
+++ b/Source/CNTKv2LibraryDll/BackCompat.cpp
@ -232,6 +232,8 @@ namespace CNTK
                    primitiveFunctionConfigParameters[PrimitiveFunction::AttributeNameOffset] = (size_t)node->As<FutureValueNode<ElementType>>()->TimeStep();
                    opType = PrimitiveOpType::FutureValue;
                }
+                else if (node->OperationName() == OperationNameOf(LogisticNode))
+                    opType = PrimitiveOpType::Logistic;
                else if (node->OperationName() == OperationNameOf(SquareErrorNode))
                    opType = PrimitiveOpType::SquaredError;
                else if (node->OperationName() == OperationNameOf(CrossEntropyWithSoftmaxNode))
--- a/Source/CNTKv2LibraryDll/Function.cpp
+++ b/Source/CNTKv2LibraryDll/Function.cpp
@ -634,10 +634,16 @@ namespace CNTK
        if (outputDataType == DataType::Unknown)
            outputDataType = firstKnownInputDataType;

-        // We currently require that the inputs' dynamic axes if any match
+        // We currently require that the inputs' dynamic axes, if any, match
        std::vector<Axis> outputDynamicAxes;
-        if ((op == PrimitiveOpType::SumAll) || (op == PrimitiveOpType::SquaredError) || (op == PrimitiveOpType::CrossEntropyWithSoftmax) || (op == PrimitiveOpType::ClassificationError))
+        if ((op == PrimitiveOpType::SumAll) ||
+            (op == PrimitiveOpType::SquaredError) ||
+            (op == PrimitiveOpType::CrossEntropyWithSoftmax) ||
+            (op == PrimitiveOpType::ClassificationError) ||
+            (op == PrimitiveOpType::Logistic))
+        {
            outputDynamicAxes = std::vector<Axis>({});
+        }
        else if (op == PrimitiveOpType::Where)
        {
            if (functionConfig.Contains(PrimitiveFunction::AttributeNameNewDynamicAxes))
@ -902,23 +908,24 @@ namespace CNTK

                NDShape outputMapCount, kernelShape;
                std::tie(outputMapCount, kernelShape) = GetConvolutionOutputMapCountAndKernelShape(inputs[0].Shape(), inputs[1].Shape());
-                    auto originalKernelShape = kernelShape;
-                    outputShape = ConvolutionOpOutputShape(op, inputs[1].Shape(), kernelShape, outputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, transpose, inferDimensions);
-                    if (originalKernelShape != kernelShape)
-                    {
-                        for (size_t i = 0; i < kernelShape.Rank(); ++i)
-                            inputs[0].m_dataFields->m_shape[i] = kernelShape[i];
-                    }
+                auto originalKernelShape = kernelShape;
+                outputShape = ConvolutionOpOutputShape(op, inputs[1].Shape(), kernelShape, outputMapCount, strides, sharing, autoPadding, lowerPad, upperPad, transpose, inferDimensions);
+                if (originalKernelShape != kernelShape)
+                {
+                    for (size_t i = 0; i < kernelShape.Rank(); ++i)
+                        inputs[0].m_dataFields->m_shape[i] = kernelShape[i];
+                }

-                    functionConfig[PrimitiveFunction::AttributeNameSharing] = AsDictionaryValueVector(sharing);
-                    functionConfig[PrimitiveFunction::AttributeNameAutoPadding] = AsDictionaryValueVector(autoPadding);
+                functionConfig[PrimitiveFunction::AttributeNameSharing] = AsDictionaryValueVector(sharing);
+                functionConfig[PrimitiveFunction::AttributeNameAutoPadding] = AsDictionaryValueVector(autoPadding);
                break;
            }
+            case PrimitiveOpType::Logistic:
            case PrimitiveOpType::SquaredError:
            case PrimitiveOpType::CrossEntropyWithSoftmax:
            case PrimitiveOpType::ClassificationError:
            {
-                if (op == PrimitiveOpType::ClassificationError)
+                if ((op == PrimitiveOpType::ClassificationError) || (op == PrimitiveOpType::Logistic))
                    assert(inputs.size() >= 2);
                else
                    assert(inputs.size() == 2);
@ -931,9 +938,9 @@ namespace CNTK
                if (predictionShape != labelsShape)
                    RuntimeError("Prediction output operand's shape %S is incompatible with label operand's shape %S for the %S operation", AsStringForErrorReporting(predictionShape).c_str(), AsStringForErrorReporting(labelsShape).c_str(), PrimitiveOpTypeName(op).c_str());

-                    std::vector<int> reductionAxes;
-                    for (int i = 0; i < (int)inputs[0].Shape().Rank(); ++i)
-                    reductionAxes.push_back(i);
+                std::vector<int> reductionAxes;
+                for (int i = 0; i < (int)inputs[0].Shape().Rank(); ++i)
+                reductionAxes.push_back(i);

                outputShape = ReductionOpOutputShape(op, predictionShape, reductionAxes, /*preserveReductionAxes =*/ false);
                break;
@ -1615,6 +1622,9 @@ namespace CNTK
            computationNodePtr = New<ConvolutionNode<ElementType>>(network->GetDeviceId(), internalNodeName, AsTensorShape(kernelShape), AsTensorShape(outputMapCount), AsTensorShape(strides), sharing, autoPadding, AsTensorShape(lowerPad), AsTensorShape(upperPad), transpose, ImageLayoutKind::CHW, maxTempMemSizeInSamples);
            break;
        }
+        case PrimitiveOpType::Logistic:
+            computationNodePtr = New<LogisticNode<ElementType>>(network->GetDeviceId(), internalNodeName);
+            break;
        case PrimitiveOpType::SquaredError:
            computationNodePtr = New<SquareErrorNode<ElementType>>(network->GetDeviceId(), internalNodeName);
            break;
@ -2794,6 +2804,18 @@ namespace CNTK
        return BinaryOp(PrimitiveOpType::TransposeTimes, leftOperand, rightOperand, std::move(additionalProperties), name);
    }

+    FunctionPtr BinaryCrossEntropy(const Variable& prediction, const Variable& targets, const std::wstring& name)
+    {
+        std::vector<Variable> operands = { prediction, targets };
+        return CompositeFunction::Create(MakeSharedObject<PrimitiveFunction>(PrimitiveOpType::Logistic, operands, Dictionary(), name), name);
+    }
+
+    FunctionPtr WeightedBinaryCrossEntropy(const Variable& prediction, const Variable& targets, const Variable& weights, const std::wstring& name)
+    {
+        std::vector<Variable> operands = { prediction, targets, weights };
+        return CompositeFunction::Create(MakeSharedObject<PrimitiveFunction>(PrimitiveOpType::Logistic, operands, Dictionary(), name), name);
+    }
+
    FunctionPtr SquaredError(const Variable& prediction, const Variable& targets, const std::wstring& name)
    {
        auto difference = Minus(prediction, targets);
--- a/Source/CNTKv2LibraryDll/Function.h
+++ b/Source/CNTKv2LibraryDll/Function.h
@ -65,7 +65,7 @@ namespace CNTK
        {PrimitiveOpType::Times, L"Times"},
        {PrimitiveOpType::TransposeTimes, L"TransposeTimes"},
        {PrimitiveOpType::Convolution, L"Convolution"},
-        {PrimitiveOpType::SquaredError, L"SquaredError"},
+        { PrimitiveOpType::SquaredError, L"SquaredError" },
        {PrimitiveOpType::CrossEntropyWithSoftmax, L"CrossEntropyWithSoftmax"},
        {PrimitiveOpType::ClassificationError, L"ClassificationError"},
        {PrimitiveOpType::PastValue, L"PastValue"},
@ -79,6 +79,7 @@ namespace CNTK
        {PrimitiveOpType::RandomSample, L"RandomSample"},
        {PrimitiveOpType::RandomSampleInclusionFrequency, L"RandomSampleInclusionFrequency"},
        {PrimitiveOpType::ROIPooling, L"ROIPooling"},
+        { PrimitiveOpType::Logistic, L"Logistic" },
    };

    inline const std::wstring& PrimitiveOpTypeName(PrimitiveOpType opType)
@ -103,7 +104,15 @@ namespace CNTK
            if (numFunctionInputs > 2)
                indexMap.insert({2, 2});
        }
-        else if ((op == PrimitiveOpType::CrossEntropyWithSoftmax) || (op == PrimitiveOpType::GatherPacked))
+        else if (op == PrimitiveOpType::Logistic)
+        {
+            indexMap = std::unordered_map<size_t, size_t>({ { 0, 1 }, { 1, 0 } });
+            if (numFunctionInputs > 2)
+                indexMap.insert({ 2, 2 });
+        }
+        else if (op == PrimitiveOpType::CrossEntropyWithSoftmax)
+            indexMap = std::unordered_map<size_t, size_t>({ { 0, 1 }, { 1, 0 } });
+        else if (op == PrimitiveOpType::GatherPacked)
            indexMap = std::unordered_map<size_t, size_t>({ { 0, 1 }, { 1, 0 } });
        else if (op == PrimitiveOpType::ScatterPacked)
            indexMap = std::unordered_map<size_t, size_t>({ { 0, 2 }, { 1, 1 }, { 2, 0 } });
--- a/Source/CNTKv2LibraryDll/PrimitiveOpType.h
+++ b/Source/CNTKv2LibraryDll/PrimitiveOpType.h
@ -57,6 +57,7 @@ namespace CNTK
        RandomSample = 45,
        RandomSampleInclusionFrequency = 46,
        ROIPooling = 47,
+        Logistic = 48,
        // New op types should only be appended to the end of this list.
    };
 }
--- a/Source/ComputationNetworkLib/LinearAlgebraNodes.h
+++ b/Source/ComputationNetworkLib/LinearAlgebraNodes.h
@ -395,7 +395,7 @@ public:
            // If input data is sparse, then gradient is block sparse.
            if (InputRef(1).Value().GetMatrixType() == SPARSE && InputRef(0).Gradient().GetMatrixType() == DENSE && Gradient().GetMatrixType() == DENSE)
            {
-                // We need a sparse matrix for the gradient. However, we should allocate a new one instead of switching the type in place
+                // We need a sparse matrix for the gradient. We allocate a new one instead of switching the type in place
                // since switching in place may affect other nodes who share this matrix due to memory sharing
                auto& currentInput0GradientMatrixRef = InputRef(0).Gradient();
                auto newInput0SparseGradientMatrix = std::make_shared<Matrix<ElemType>>(currentInput0GradientMatrixRef.GetNumRows(),
@ -556,7 +556,7 @@ public:
        {
            Input(0)->CreateGradientMatrixIfNull();

-            // We need a sparse matrix for the gradient. However, we should allocate a new one instead of switching the type in place
+            // We need a sparse matrix for the gradient. We allocate a new one instead of switching the type in place
            // since switching in place may affect other nodes who share this matrix due to memory sharing
            auto& currentInput0GradientMatrixRef = InputRef(0).Gradient();
            if (currentInput0GradientMatrixRef.GetMatrixType() != SPARSE)
--- a/Source/ComputationNetworkLib/TrainingNodes.cpp
+++ b/Source/ComputationNetworkLib/TrainingNodes.cpp
@ -126,7 +126,7 @@ void RandomSampleNode<ElemType>::ForwardPropNonLooping()
    if (ValueAsMatrix().GetMatrixType() != SPARSE)
    {
        // BUGBUG: matrix type should be configured during validation
-        // We should allocate a new one instead of switching the type in place since switching in place may
+        // Note: We allocate a new one instead of switching the type in place since switching in place may
        // affect other nodes who share this matrix due to memory sharing
        auto newSparseValueMatrix = std::make_shared<Matrix<ElemType>>(ValueAsMatrix().GetNumRows(), ValueAsMatrix().GetNumCols(), CPUDEVICE, SPARSE, matrixFormatSparseCSC);
 #ifdef _MSC_VER
@ -140,10 +140,7 @@ void RandomSampleNode<ElemType>::ForwardPropNonLooping()

    // TODO: Should we prepare the CSC data directly on the CPU and move it in one go?
    // Currently the reader will place the data onto the GPU. It will then be pulled on-demand to the CPU once (and cached there).
-    valueMatrix.TransferToDeviceIfNotThere(CPUDEVICE, /*ismoved =*/ true/*means: BOTH state not ok */, /*emptyTransfer =*/ true, /*updatePreferredDevice =*/ false);
-
-    // BUGUBUG: This is a no-op; was the intent to change the preferred device to CPU?
-    valueMatrix.SetDevice(CPUDEVICE);
+    valueMatrix.TransferToDeviceIfNotThere(CPUDEVICE, /*ismoved =*/ true/*means: BOTH state not ok */, /*emptyTransfer =*/ true, /*updatePreferredDevice =*/ true);
    valueMatrix.Reset();

    // Get vector with indices of randomly sampled classes
--- a/Tests/EndToEndTests/EvalClientTests/CPPEvalExtendedClientTest/CPPEvalExtendedClientTest.vcxproj
+++ b/Tests/EndToEndTests/EvalClientTests/CPPEvalExtendedClientTest/CPPEvalExtendedClientTest.vcxproj
@ -5,6 +5,10 @@
      <Configuration>Debug</Configuration>
      <Platform>x64</Platform>
    </ProjectConfiguration>
+    <ProjectConfiguration Include="Release_NoOpt|x64">
+      <Configuration>Release_NoOpt</Configuration>
+      <Platform>x64</Platform>
+    </ProjectConfiguration>
    <ProjectConfiguration Include="Release|x64">
      <Configuration>Release</Configuration>
      <Platform>x64</Platform>
--- a/Tests/EndToEndTests/EvalClientTests/CSEvalClientTest/CSEvalClientTest.csproj
+++ b/Tests/EndToEndTests/EvalClientTests/CSEvalClientTest/CSEvalClientTest.csproj
@ -52,7 +52,6 @@
    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
  </PropertyGroup>
  <PropertyGroup Condition="'$(Configuration)|$(Platform)' == 'Release_NoOpt|x64'">
-    <OutputPath>bin\x64\Release_NoOpt\</OutputPath>
    <DefineConstants>TRACE</DefineConstants>
    <Optimize>true</Optimize>
    <TreatWarningsAsErrors>true</TreatWarningsAsErrors>
@ -60,7 +59,6 @@
    <PlatformTarget>x64</PlatformTarget>
    <ErrorReport>prompt</ErrorReport>
    <CodeAnalysisRuleSet>MinimumRecommendedRules.ruleset</CodeAnalysisRuleSet>
-    <Prefer32Bit>true</Prefer32Bit>
  </PropertyGroup>
  <ItemGroup>
    <Reference Include="System" />
--- a/Tests/Install/linux/test.sh
+++ b/Tests/Install/linux/test.sh
@ -3,7 +3,7 @@ set -x -e -o pipefail

 USAGE="Usage: $0 <drops-to-test>"

-REPO_TAG=v2.0.beta2.0
+REPO_TAG=v2.0.beta3.0

 while [ $# -gt 0 ]; do
  case "$1" in
--- a/bindings/python/cntk/graph.py
+++ b/bindings/python/cntk/graph.py
@ -1,4 +1,5 @@
 # Copyright (c) Microsoft. All rights reserved.
+
 # Licensed under the MIT license. See LICENSE.md file in the project root
 # for full license information.
 # ==============================================================================
@ -13,7 +14,6 @@ def dfs_walk(node, visitor):
        node (graph node): the node to start the journey from
        visitor (Python function or lambda): function that takes a node as
         argument and returns ``True`` if that node should be returned.
-
    Returns:
        List of nodes, for which ``visitor`` was ``True``
    '''
@ -48,13 +48,112 @@ def find_nodes_by_name(node, node_name):
    '''
    Finds nodes in the graph starting from `node` and doing a depth-first
    search.
-
    Args:
        node (graph node): the node to start the journey from
        node_name (`str`): name for which we are search nodes
-
    Returns:
        List of nodes having the specified name
    '''
    return dfs_walk(node, lambda x: x.name == node_name)

+def output_function_graph(node,dot_file_path=None,png_file_path=None):
+    '''
+    Walks through every node of the graph starting at ``node``,
+    creates a network graph, and saves it as a string. If dot_file_name or 
+    png_file_name specified corresponding files will be saved.
+    
+    
+    Requirements for DOT output: pydot_ng
+    Requirements for PNG output: pydot_ng and graphviz
+
+    Args:
+        node (graph node): the node to start the journey from
+        dot_file_path (`str`, optional): DOT file path
+        png_file_path (`str`, optional): PNG file path
+
+    Returns:
+        `str` containing all nodes and edges
+    '''
+
+    dot = (dot_file_path != None)
+    png = (png_file_path != None)
+
+    if (dot or png):
+
+        try:
+            import pydot_ng as pydot
+        except ImportError:
+            raise ImportError("PNG and DOT format requires pydot_ng package. Unable to import pydot_ng.")
+
+        # initialize a dot object to store vertices and edges
+        dot_object = pydot.Dot(graph_name="network_graph",rankdir='TB')
+        dot_object.set_node_defaults(shape='rectangle', fixedsize='false',
+                                 height=.85, width=.85, fontsize=12)
+        dot_object.set_edge_defaults(fontsize=10)
+    
+    # string to store model 
+    model = ''
+
+    # walk every node of the graph iteratively
+    visitor = lambda x: True
+    stack = [node]
+    accum = []
+    visited = set()
+
+    while stack:
+        node = stack.pop()
+        
+        if node in visited:
+            continue
+
+        try:
+            # Function node
+            node = node.root_function
+            stack.extend(node.inputs)
+
+            # add current node
+            model += node.op_name + '('
+            if (dot or png):
+                cur_node = pydot.Node(node.op_name+' '+node.uid,label=node.op_name,shape='circle',
+                                        fixedsize='true', height=1, width=1)
+                dot_object.add_node(cur_node)
+
+            # add node's inputs
+            for i in range(len(node.inputs)):
+                child = node.inputs[i]
+                
+                model += child.uid
+                if (i != len(node.inputs) - 1):
+                    model += ", "
+
+                if (dot or png):
+                    child_node = pydot.Node(child.uid)
+                    dot_object.add_node(child_node)
+                    dot_object.add_edge(pydot.Edge(child_node, cur_node,label=str(child.shape)))
+
+            # ad node's output
+            model += ") -> " + node.outputs[0].uid +'\n'
+
+            if (dot or png):
+                out_node = pydot.Node(node.outputs[0].uid)
+                dot_object.add_node(out_node)
+                dot_object.add_edge(pydot.Edge(cur_node,out_node,label=str(node.outputs[0].shape)))
+
+        except AttributeError:
+            # OutputVariable node
+            try:
+                if node.is_output:
+                    stack.append(node.owner)
+            except AttributeError:
+                pass
+
+    if visitor(node):
+        accum.append(node)
+
+    if (png):
+        dot_object.write_png(png_file_path, prog='dot')
+    if (dot):
+        dot_object.write_raw(dot_file_path)
+
+    # return lines in reversed order
+    return "\n".join(model.split("\n")[::-1])
--- a/bindings/python/cntk/ops/init.py
+++ b/bindings/python/cntk/ops/init.py
@ -59,9 +59,52 @@ def alias(x, name=''):
    return alias(x, name)

 ##########################################################################
-# evaluation ops
+# loss and evaluation ops
 ##########################################################################

+@typemap
+def binary_cross_entropy(output, target, name=''):
+    r'''
+    This operation computes the binary cross entropy between the ``output`` and ``target``.
+
+    Example:
+        TBA
+
+    Args:
+        output: the computed posterior probability from the network
+        target: ground-truth label, 0 or 1
+        name (`str`, optional): the name of the Function instance in the network
+    Returns:
+        :class:`cntk.ops.functions.Function`
+    '''
+    from cntk.cntk_py import binary_cross_entropy
+    dtype = get_data_type(output, target)
+    output = sanitize_input(output, dtype)
+    target = sanitize_input(target, dtype)
+    return binary_cross_entropy(output, target, name)
+
+@typemap
+def weighted_binary_cross_entropy(output, target, weight, name=''):
+    r'''
+    This operation computes the weighted binary cross entropy between the ``output`` and ``target``.
+
+    Example:
+        TBA
+
+    Args:
+        output: the computed posterior probability from the network
+        target: ground-truth label, 0 or 1
+        weight: weight of each example
+        name (`str`, optional): the name of the Function instance in the network
+    Returns:
+        :class:`cntk.ops.functions.Function`
+    '''
+    from cntk.cntk_py import weighted_binary_cross_entropy
+    dtype = get_data_type(output, target, weight)
+    output = sanitize_input(output, dtype)
+    target = sanitize_input(target, dtype)
+    weight = sanitize_input(weight, dtype)
+    return weighted_binary_cross_entropy(output, target, weight, name)

@typemap
 def cross_entropy_with_softmax(output_vector, target_vector, axis=-1, name=''):
--- a/bindings/python/cntk/tests/graph_test.py
+++ b/bindings/python/cntk/tests/graph_test.py
@ -41,6 +41,18 @@ def _graph_dict():
    d['root'] = d['first']

    return d
+
+def _simple_dict():
+    d = {}
+
+    d['i1'] = input_variable(shape=(2,3), name='i1')
+    d['i2'] = input_variable(shape=(2,3), name='i2')
+    d['p1'] = parameter(shape=(3,2), name='p1')
+    d['op1'] = plus(d['i1'], d['i2'], name='op1')
+    d['op2'] = times(d['op1'], d['p1'], name='op2')
+    d['root'] = d['op2']
+
+    return d
    

 def test_find_nodes():
@ -57,3 +69,16 @@ def test_find_nodes():

    none = find_nodes_by_name(d['root'], 'none')
    assert none == []
+
+
+def test_output_funtion_graph():
+    d = _simple_dict()
+
+    m = output_function_graph(d['root'])
+    p = "\nPlus"
+    t = "\nTimes"
+
+    assert len(m) != 0
+    assert p in m
+    assert t in m
+    assert m.find(p) < m.find(t)
--- a/bindings/python/doc/conf.py
+++ b/bindings/python/doc/conf.py
@ -63,9 +63,9 @@ author = 'Microsoft'
 # built documents.
 #
 # The short X.Y version.
-version = '2.0.beta2.0'
+version = '2.0.beta3.0'
 # The full version, including alpha/beta/rc tags.
-release = '2.0.beta2.0'
+release = '2.0.beta3.0'

 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
--- a/bindings/python/doc/examples.rst
+++ b/bindings/python/doc/examples.rst
@ -5,27 +5,27 @@ The best way to learn about the APIs currently is to look at the
 following examples in the [CNTK clone root]/bindings/python/examples
 directory:

-  `MNIST <https://github.com/Microsoft/CNTK/blob/v2.0.beta2.0/bindings/python/examples/MNIST/SimpleMNIST.py>`__:
+-  `MNIST <https://github.com/Microsoft/CNTK/blob/v2.0.beta3.0/bindings/python/examples/MNIST/SimpleMNIST.py>`__:
   A fully connected feed-forward model for classification of MNIST
   images. (follow the instructions in
   Examples/Image/DataSets/MNIST/README.md)

-  `CifarResNet <https://github.com/Microsoft/CNTK/blob/v2.0.beta2.0/bindings/python/examples/CifarResNet/CifarResNet.py>`__:
+-  `CifarResNet <https://github.com/Microsoft/CNTK/blob/v2.0.beta3.0/bindings/python/examples/CifarResNet/CifarResNet.py>`__:
   An image classification ResNet model for training on the CIFAR image
   dataset. (follow the instructions in
   Examples/Image/DataSets/CIFAR-10/README.md to get the CIFAR dataset
   and convert it to the CNTK supported format)

-  `SequenceClassification <https://github.com/Microsoft/CNTK/blob/v2.0.beta2.0/bindings/python/examples/SequenceClassification/SequenceClassification.py>`__:
+-  `SequenceClassification <https://github.com/Microsoft/CNTK/blob/v2.0.beta3.0/bindings/python/examples/SequenceClassification/SequenceClassification.py>`__:
   An LSTM sequence classification model for text data.

-  `Sequence2Sequence <https://github.com/Microsoft/CNTK/blob/v2.0.beta2.0/bindings/python/examples/Sequence2Sequence/Sequence2Sequence.py>`__:
+-  `Sequence2Sequence <https://github.com/Microsoft/CNTK/blob/v2.0.beta3.0/bindings/python/examples/Sequence2Sequence/Sequence2Sequence.py>`__:
   A sequence to sequence grapheme to phoneme translation model that
   trains on the CMUDict corpus.

-  `NumpyInterop <https://github.com/Microsoft/CNTK/blob/v2.0.beta2.0/bindings/python/examples/NumpyInterop/FeedForwardNet.py>`__
+-  `NumpyInterop <https://github.com/Microsoft/CNTK/blob/v2.0.beta3.0/bindings/python/examples/NumpyInterop/FeedForwardNet.py>`__
   - NumPy interoperability example showing how to train a simple feed-forward
   network with training data fed using NumPy arrays.

-  `LanguageUnderstanding <https://github.com/Microsoft/CNTK/blob/v2.0.beta2.0/bindings/python/examples/LanguageUnderstanding/LanguageUnderstanding.py>`__
+-  `LanguageUnderstanding <https://github.com/Microsoft/CNTK/blob/v2.0.beta3.0/bindings/python/examples/LanguageUnderstanding/LanguageUnderstanding.py>`__
   - Language Understanding.
--- a/bindings/python/doc/index.rst
+++ b/bindings/python/doc/index.rst
@ -2,7 +2,7 @@
 .. some aliases
 .. _CNTK: http://cntk.ai/

-Python API for CNTK (2.0.beta2.0)
+Python API for CNTK (2.0.beta3.0)
 ===============================

 CNTK_, the Microsoft Cognitive Toolkit, is a system for describing, training,
@ -12,7 +12,7 @@ neural networks (CNNs), recurrent neural networks (RNNs), long short term
 memory (LSTM), logistic regression, and maximum entropy model. CNTK is an 
 implementation of computational networks that supports both CPU and GPU.
 
-This page describes the Python API for CNTK_ version 2.0.beta2.0. This is an ongoing effort
+This page describes the Python API for CNTK_ version 2.0.beta3.0. This is an ongoing effort
 to expose such an API to the CNTK system, thus enabling the use of higher-level
 tools such as IDEs to facilitate the definition of computational networks, to execute
 them on sample data in real time.
--- a/bindings/python/doc/tutorials.rst
+++ b/bindings/python/doc/tutorials.rst
@ -17,12 +17,12 @@ Tutorials

 #.  CNTK 203: `Reinforcement learning basics`_ with OpenAI Gym data
 	
-.. _`Logistic Regression`: https://github.com/Microsoft/CNTK/tree/v2.0.beta2.0/bindings/python/tutorials/CNTK_101_LogisticRegression.ipynb
-.. _`Feed Forward Network`: https://github.com/Microsoft/CNTK/tree/v2.0.beta2.0/bindings/python/tutorials/CNTK_102_FeedForward.ipynb
-.. _`MNIST data preparation`: https://github.com/Microsoft/CNTK/tree/v2.0.beta2.0/bindings/python/tutorials/CNTK_103A_MNIST_DataLoader.ipynb
-.. _`Feed Forward Classifier`: https://github.com/Microsoft/CNTK/tree/v2.0.beta2.0/bindings/python/tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb
-.. _`CIFAR-10 Data preparation`: https://github.com/Microsoft/CNTK/tree/v2.0.beta2.0/bindings/python/tutorials/CNTK_201A_CIFAR-10_DataLoader.ipynb
-.. _`VGG and ResNet classifiers`: https://github.com/Microsoft/CNTK/tree/v2.0.beta2.0/bindings/python/tutorials/CNTK_201B_CIFAR-10_ImageHandsOn.ipynb
-.. _`Language understanding`: https://github.com/Microsoft/CNTK/blob/v2.0.beta2.0/bindings/python/tutorials/CNTK_202_Language_Understanding.ipynb
+.. _`Logistic Regression`: https://github.com/Microsoft/CNTK/tree/v2.0.beta3.0/bindings/python/tutorials/CNTK_101_LogisticRegression.ipynb
+.. _`Feed Forward Network`: https://github.com/Microsoft/CNTK/tree/v2.0.beta3.0/bindings/python/tutorials/CNTK_102_FeedForward.ipynb
+.. _`MNIST data preparation`: https://github.com/Microsoft/CNTK/tree/v2.0.beta3.0/bindings/python/tutorials/CNTK_103A_MNIST_DataLoader.ipynb
+.. _`Feed Forward Classifier`: https://github.com/Microsoft/CNTK/tree/v2.0.beta3.0/bindings/python/tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb
+.. _`CIFAR-10 Data preparation`: https://github.com/Microsoft/CNTK/tree/v2.0.beta3.0/bindings/python/tutorials/CNTK_201A_CIFAR-10_DataLoader.ipynb
+.. _`VGG and ResNet classifiers`: https://github.com/Microsoft/CNTK/tree/v2.0.beta3.0/bindings/python/tutorials/CNTK_201B_CIFAR-10_ImageHandsOn.ipynb
+.. _`Language understanding`: https://github.com/Microsoft/CNTK/blob/v2.0.beta3.0/bindings/python/tutorials/CNTK_202_Language_Understanding.ipynb
 .. _`Reinforcement learning basics`: https://github.com/Microsoft/CNTK/blob/master/bindings/python/tutorials/CNTK_203_Reinforcement_Learning_Basics.ipynb
  
--- a/bindings/python/setup.py
+++ b/bindings/python/setup.py
@ -165,7 +165,7 @@ else:
    kwargs = dict(package_data = package_data)

 setup(name="cntk",
-      version="2.0.beta2.0",
+      version="2.0.beta3.0",
      url="http://cntk.ai",
      ext_modules=[cntk_module],
      packages=packages,
--- a/bindings/python/tutorials/CNTK_101_LogisticRegression.ipynb
+++ b/bindings/python/tutorials/CNTK_101_LogisticRegression.ipynb
@ -10,7 +10,7 @@
    "\n",
    "This tutorial is targeted to individuals who are new to CNTK and to machine learning. In this tutorial, you will train a simple yet powerful machine learning model that is widely used in industry for a variety of applications. The model trained below scales to massive data sets in the most expeditious manner by harnessing computational scalability leveraging the computational resources you may have (one or more CPU cores, one or more GPUs, a cluster of CPUs or a cluster of GPUs), transparently via the CNTK library.\n",
    "\n",
-    "The following notebook users Python APIs. If you are looking for this example in Brainscript, please look [here](https://github.com/Microsoft/CNTK/tree/v2.0.beta2.0/Examples/Tutorials/LogisticRegressionAndMultiClass). \n",
+    "The following notebook users Python APIs. If you are looking for this example in Brainscript, please look [here](https://github.com/Microsoft/CNTK/tree/v2.0.beta3.0/Examples/Tutorials/LogisticRegressionAndMultiClass). \n",
    "\n",
    "## Introduction\n",
    "\n",
--- a/bindings/python/tutorials/CNTK_102_FeedForward.ipynb
+++ b/bindings/python/tutorials/CNTK_102_FeedForward.ipynb
@ -767,7 +767,7 @@
    "\n",
    "If you want to try running the tutorial from python command prompt. Please run the [FeedForwardNet.py][] example.\n",
    "\n",
-    "[FeedForwardNet.py]: https://github.com/Microsoft/CNTK/blob/v2.0.beta2.0/bindings/python/examples/NumpyInterop/FeedForwardNet.py"
+    "[FeedForwardNet.py]: https://github.com/Microsoft/CNTK/blob/v2.0.beta3.0/bindings/python/examples/NumpyInterop/FeedForwardNet.py"
   ]
  },
  {
--- a/bindings/python/tutorials/CNTK_103A_MNIST_DataLoader.ipynb
+++ b/bindings/python/tutorials/CNTK_103A_MNIST_DataLoader.ipynb
@ -12,7 +12,7 @@
    "\n",
    "CNTK 103 tutorial is divided into two parts:\n",
    "- Part A: Familiarize with the [MNIST][] database that will be used later in the tutorial\n",
-    "- [Part B](https://github.com/Microsoft/CNTK/blob/v2.0.beta2.0/bindings/python/tutorials/CNTK_103A_MNIST_DataLoader.ipynb): We will use the feedforward classifier used in CNTK 102 to classify digits in MNIST data set.\n",
+    "- [Part B](https://github.com/Microsoft/CNTK/blob/v2.0.beta3.0/bindings/python/tutorials/CNTK_103A_MNIST_DataLoader.ipynb): We will use the feedforward classifier used in CNTK 102 to classify digits in MNIST data set.\n",
    "\n",
    "[MNIST]: http://yann.lecun.com/exdb/mnist/\n",
    "\n"
--- a/bindings/python/tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb
+++ b/bindings/python/tutorials/CNTK_103B_MNIST_FeedForwardNetwork.ipynb
@ -12,7 +12,7 @@
    "\n",
    "We assume that you have successfully completed CNTK 103 Part A.\n",
    "\n",
-    "In this tutorial we will train a fully connected network on MNIST data. This notebook provides the recipe using Python APIs. If you are looking for this example in Brainscript, please look [here](https://github.com/Microsoft/CNTK/tree/v2.0.beta2.0/Examples/Image/GettingStarted)\n",
+    "In this tutorial we will train a fully connected network on MNIST data. This notebook provides the recipe using Python APIs. If you are looking for this example in Brainscript, please look [here](https://github.com/Microsoft/CNTK/tree/v2.0.beta3.0/Examples/Image/GettingStarted)\n",
    "\n",
    "## Introduction\n",
    "\n",
@ -765,7 +765,7 @@
   "source": [
    "#### Code link\n",
    "\n",
-    "If you want to try running the tutorial from python command prompt. Please run the [SimpleMNIST.py](https://github.com/Microsoft/CNTK/tree/v2.0.beta2.0/bindings/python/examples/MNIST) example."
+    "If you want to try running the tutorial from python command prompt. Please run the [SimpleMNIST.py](https://github.com/Microsoft/CNTK/tree/v2.0.beta3.0/bindings/python/examples/MNIST) example."
   ]
  },
  {