diff --git a/Source/CNTK/BrainScript/ExperimentalNetworkBuilder.cpp b/Source/CNTK/BrainScript/ExperimentalNetworkBuilder.cpp index 0868af87b..328c17ea6 100644 --- a/Source/CNTK/BrainScript/ExperimentalNetworkBuilder.cpp +++ b/Source/CNTK/BrainScript/ExperimentalNetworkBuilder.cpp @@ -79,6 +79,7 @@ L"ParameterTensor(dims, learningRateMultiplier = 1.0, init = 'uniform'/*|fixedVa #ifdef COMING_SOON TernaryStandardNode(CRF, labelVectorSequence, positionDependenScoreVectorSequence, transitionScores) // TODO: better names #endif + UnaryStandardNode(Abs, x) QuaternaryStandardNode(ClassBasedCrossEntropyWithSoftmax, labelClassDescriptorVectorSequence, mainInputInfo, mainWeight, classLogProbsBeforeSoftmax) // BUGBUG: the commented-out ones are not mentioned in the CNTK book, nor are their parameters documented in the source code BinaryStandardNode(ColumnElementTimes, aVectorSequence, anotherVectorSequence) diff --git a/Source/CNTK/NetworkDescriptionLanguage.cpp b/Source/CNTK/NetworkDescriptionLanguage.cpp index 46eddea3e..01177c0b3 100644 --- a/Source/CNTK/NetworkDescriptionLanguage.cpp +++ b/Source/CNTK/NetworkDescriptionLanguage.cpp @@ -149,7 +149,8 @@ bool CheckFunction(std::string& p_nodeType, bool* allowUndeterminedVariable) wstring nodeType = msra::strfun::utf16(p_nodeType); bool ret = false; - if (EqualInsensitive(nodeType, OperationNameOf(AveragePoolingNode))) ret = true; + if (EqualInsensitive(nodeType, OperationNameOf(AbsNode))) ret = true; + else if (EqualInsensitive(nodeType, OperationNameOf(AveragePoolingNode))) ret = true; else if (EqualInsensitive(nodeType, OperationNameOf(BatchNormalizationNode))) ret = true; #ifdef COMING_SOON else if (EqualInsensitive(nodeType, OperationNameOf(CRFNode), L"CRF")) ret = true; diff --git a/Source/ComputationNetworkLib/ComputationNetworkBuilder.cpp b/Source/ComputationNetworkLib/ComputationNetworkBuilder.cpp index a4a962f50..82c37b342 100644 --- a/Source/ComputationNetworkLib/ComputationNetworkBuilder.cpp +++ b/Source/ComputationNetworkLib/ComputationNetworkBuilder.cpp @@ -37,7 +37,8 @@ static shared_ptr> CreateStandardNode(const std::wstri if (nodeType == OperationNameOf(CRFNode)) return New>(forward<_Types>(_Args)...); else #endif - if (nodeType == OperationNameOf(ClassBasedCrossEntropyWithSoftmaxNode))return New>(forward<_Types>(_Args)...); + if (nodeType == OperationNameOf(AbsNode)) return New>(forward<_Types>(_Args)...); + else if (nodeType == OperationNameOf(ClassBasedCrossEntropyWithSoftmaxNode))return New>(forward<_Types>(_Args)...); else if (nodeType == OperationNameOf(CosDistanceNode)) return New>(forward<_Types>(_Args)...); else if (nodeType == OperationNameOf(CosDistanceWithNegativeSamplesNode)) return New>(forward<_Types>(_Args)...); else if (nodeType == OperationNameOf(CosineNode)) return New>(forward<_Types>(_Args)...); diff --git a/Source/ComputationNetworkLib/ComputationNetworkBuilder.h b/Source/ComputationNetworkLib/ComputationNetworkBuilder.h index 1062cf5c2..1f6cfc94f 100644 --- a/Source/ComputationNetworkLib/ComputationNetworkBuilder.h +++ b/Source/ComputationNetworkLib/ComputationNetworkBuilder.h @@ -77,6 +77,7 @@ public: #ifdef COMING_SOON ComputationNodePtr CRF(const ComputationNodePtr label, const ComputationNodePtr postDepScore, const ComputationNodePtr transition_score, const std::wstring nodeName = L""); #endif + ComputationNodePtr Abs(const ComputationNodePtr a, const std::wstring nodeName = L""); ComputationNodePtr ClassCrossEntropyWithSoftmax(const ComputationNodePtr label, const ComputationNodePtr prediction, const ComputationNodePtr input_weight, const ComputationNodePtr cls_log_post_prob, const std::wstring nodeName = L""); ComputationNodePtr Cos(const ComputationNodePtr a, const std::wstring nodeName = L""); ComputationNodePtr CosDistance(const ComputationNodePtr a, const ComputationNodePtr b, const std::wstring nodeName = L""); diff --git a/Source/ComputationNetworkLib/NonlinearityNodes.h b/Source/ComputationNetworkLib/NonlinearityNodes.h index f22b43516..983ea5946 100644 --- a/Source/ComputationNetworkLib/NonlinearityNodes.h +++ b/Source/ComputationNetworkLib/NonlinearityNodes.h @@ -117,6 +117,7 @@ DeclareUnaryElementWiseWithOpCodeNode(RectifiedLinear, LinearRectifier, Elementw DeclareUnaryElementWiseWithOpCodeNode(Log, Log, ElementwiseProductWithLogDerivativeFromOutput, true); DeclareUnaryElementWiseWithOpCodeNode(Exp, Exp, ElementwiseProduct, true); DeclareUnaryElementWiseWithOpCodeNode(Cosine, Cosine, ElementwiseProductWithCosDerivative, false); +DeclareUnaryElementWiseWithOpCodeNode(Abs, Abs, ElementwiseProductWithAbsDerivative, false); #pragma pop_macro("DeclareUnaryElementWiseWithOpCodeNode") diff --git a/Source/Math/CommonMatrix.h b/Source/Math/CommonMatrix.h index 16c2b4f4b..a81358930 100644 --- a/Source/Math/CommonMatrix.h +++ b/Source/Math/CommonMatrix.h @@ -111,6 +111,7 @@ enum ElementWiseOperator opElementwiseProductWithLinearRectifierDerivativeFromOutput, opElementwiseProductWithLogDerivativeFromOutput, opElementwiseProductWithCosDerivative, + opElementwiseProductWithAbsDerivative, opSqrOfDifference, // binary ops for indexing // opIndex, @@ -160,8 +161,9 @@ enum ElementWiseOperator Macro(ElementwiseProductWithTanhDerivativeFromOutput); \ Macro(ElementwiseProductWithLinearRectifierDerivativeFromOutput); \ Macro(ElementwiseProductWithLogDerivativeFromOutput); \ - Macro(ElementwiseProductWithCosDerivative); \ - Macro(SqrOfDifference); \ + Macro(ElementwiseProductWithCosDerivative); \ + Macro(ElementwiseProductWithAbsDerivative); \ + Macro(SqrOfDifference); \ //Macro(Index); #define ForAllTernaryOps(Macro) \ diff --git a/Source/Math/TensorOps.h b/Source/Math/TensorOps.h index f928c604c..a400a9dfd 100644 --- a/Source/Math/TensorOps.h +++ b/Source/Math/TensorOps.h @@ -96,6 +96,14 @@ DECL ElemType LinearRectifierDerivative(ElemType z) return z > 0 ? (ElemType) 1 : 0; } +template +DECL ElemType Sgn(ElemType z) +{ + if (z > 0.0) return 1.0; + if (z < 0.0) return -1.0; + return z; +} + template DECL ElemType Sqr(ElemType z) { @@ -227,6 +235,7 @@ DefBinaryOp(ElementwiseProductWithTanhDerivativeFromOutput, a*(1 - b * b)); DefBinaryOp(ElementwiseProductWithLinearRectifierDerivativeFromOutput, b > 0 ? a : 0); DefBinaryOp(ElementwiseProductWithLogDerivativeFromOutput, a* exp_(-b)); DefBinaryOp(ElementwiseProductWithCosDerivative, a * -sin_(b)); // note: b = input for cos() +DefBinaryOp(ElementWideProductWithAbsDerivative, a * Sgn(b)); // note: b = input for abs() DefBinaryOp(SqrOfDifference, Sqr(a - b)); //DefBinaryOp(Index, IndexElement(a, b, i)); // note: this one uses the third argument