added new tensor operations: And, Or, Xor, ElementwiseProductWithSigmoidDerivative (SigmoidDerivative is only ever used in this context), and Clip

Source/ComputationNetworkLib/LinearAlgebraNodes.h

@@ -6,10 +6,10 @@
 #pragma once
 
 #include "Basics.h"
-#include "Matrix.h"
-#include "TensorView.h"
 #include "ComputationNode.h"
 #include "ConvolutionalNodes.h"
+#include "Matrix.h"
+#include "TensorView.h"
 
 #include <unordered_set>
 #include <map>

Source/ComputationNetworkLib/NonlinearityNodes.h

@@ -5,6 +5,11 @@
 //
 #pragma once
 
+#include "Basics.h"
+#include "ComputationNode.h"
+#include "Matrix.h"
+#include "TensorView.h"
+
 #include <unordered_set>
 #include <map>
 #include <string>
@@ -18,10 +23,6 @@
 #include <sstream>
 #include <iostream>
 
-#include "Basics.h"
-#include "Matrix.h"
-#include "ComputationNode.h"
-
 namespace Microsoft { namespace MSR { namespace CNTK {
 
     // -----------------------------------------------------------------------
@@ -204,8 +205,10 @@ namespace Microsoft { namespace MSR { namespace CNTK {
 
         /*virtual*/ void BackpropToV(TensorView<ElemType> gradient, const TensorView<ElemType>& inputFunctionValues, TensorView<ElemType> inputGradientValues, const TensorView<ElemType>& gradientValues, const TensorView<ElemType>& functionValues)
         {
-            gradient.AssignSigmoidDerivativeOf(inputFunctionValues);
-            inputGradientValues.AddElementwiseProductOf(gradientValues, gradient);
+            //gradient.AssignSigmoidDerivativeOf(inputFunctionValues);
+            //inputGradientValues.AddElementwiseProductOf(gradientValues, gradient);
+            //gradient.AssignSigmoidDerivativeOf(inputFunctionValues);
+            inputGradientValues.AddElementwiseProductWithSigmoidDerivativeOf(gradientValues, inputFunctionValues);
         }
 
         virtual bool OutputUsedInComputingInputNodesGradients() const override { return false; }

Source/Math/CommonMatrix.h

@@ -55,16 +55,16 @@ namespace Microsoft { namespace MSR { namespace CNTK {
         opNegate, opNot,
         opAbs,
         opSigmoid, opSigmoidDerivative, opTanh, opSqrt, opExp, opLog, opLinearRectifierDerivative, opCosine, opNegativeSine,
-        // these are not implemented yet:
-        opSaturateBetaAlpha, opSumAlpha, opSubDifferenceToAlpha, opSubDifferenceFromAlpha,
         // binary
         opSum, opDifference, opElementwiseProduct, opElementwiseQuotient,
         opLogSum, opMax, opMin,
         opEQ, opNE, opGT, opLT, opGE, opLE,
+        opAnd, opOr, opXor,
         opMaskNegative,
+        opElementwiseProductWithSigmoidDerivative/* a * dsigmoid/dx(b) */,
         // ternary
-        opCond
-        // Note: not all of the above are actually implement at present; and not all that's implemented has an opcode.
+        opCond/*a ? b : c*/, opClip/*clip a within interval b..c*/
+        // Note: not all that's implemented in CNTK ComputationNodes has an opcode yet.
     };
 
     // helper to apply a C macro for all operations of each kind
@@ -77,17 +77,16 @@ namespace Microsoft { namespace MSR { namespace CNTK {
     Macro(Abs); \
     Macro(Sigmoid); Macro(SigmoidDerivative); Macro(Tanh); Macro(Sqrt); Macro(Exp); Macro(Log); Macro(LinearRectifierDerivative); Macro(Cosine); Macro(NegativeSine);
 
-#define ForAllParameterizedUnaryOps(Macro) \
-    Macro(SaturateBetaAlpha); Macro(SumAlpha); Macro(SubDifferenceToAlpha); Macro(SubDifferenceFromAlpha);
-
 #define ForAllBinaryOps(Macro) \
     Macro(Sum); Macro(Difference); Macro(ElementwiseProduct); Macro(ElementwiseQuotient); \
     Macro(LogSum); Macro(Max); Macro(Min); \
     Macro(EQ); Macro(NE); Macro(GT); Macro(LT); Macro(GE); Macro(LE); \
-    Macro(MaskNegative);
+    Macro(And); Macro(Or); Macro(Xor);\
+    Macro(MaskNegative); \
+    Macro(ElementwiseProductWithSigmoidDerivative);
 
 #define ForAllTernaryOps(Macro) \
-    Macro(Cond);
+    Macro(Cond); Macro(Clip);
 
     // -----------------------------------------------------------------------
     // various enums to describe 

Source/Math/TensorOps.h

@@ -126,22 +126,21 @@ namespace Microsoft { namespace MSR { namespace CNTK {
     DefUnaryOp(Sigmoid, Sigmoid(a)); DefUnaryOp(SigmoidDerivative, SigmoidDerivative(a)); DefUnaryOp(Tanh, tanh_(a)); DefUnaryOp(Sqrt, Sqrt(a)); DefUnaryOp(Exp, exp_(a)); DefUnaryOp(Log, log_(a)); DefUnaryOp(LinearRectifierDerivative, LinearRectifierDerivative(a)); DefUnaryOp(Cosine, cos_(a)); DefUnaryOp(NegativeSine, -sin_(a));
 #pragma pop_macro("DefUnaryOp")
 
-    // parameterized unary ops
-    //DefUnaryOp(SaturateBetaAlpha); DefUnaryOp(SumAlpha); DefUnaryOp(SubDifferenceToAlpha); DefUnaryOp(SubDifferenceFromAlpha);
-
 #pragma push_macro("DefBinaryOp")
     #define DefBinaryOp(op, expr) template<class ElemType> DECL ElemType Op ## op(ElemType a, ElemType b) { return expr; }
 
     DefBinaryOp(Sum, a + b); DefBinaryOp(Difference, a - b); DefBinaryOp(ElementwiseProduct, a * b); DefBinaryOp(ElementwiseQuotient, a / b);
     DefBinaryOp(LogSum, LogAdd(a, b)); DefBinaryOp(Max, a > b ? a : b); DefBinaryOp(Min, a < b ? a : b);
     DefBinaryOp(EQ, a == b); DefBinaryOp(NE, a != b); DefBinaryOp(GT, a > b); DefBinaryOp(LT, a < b); DefBinaryOp(GE, a >= b); DefBinaryOp(LE, a <= b);
+    DefBinaryOp(And, (float)((!!a) && (!!b))); DefBinaryOp(Or, (float)((!!a) || (!!b))); DefBinaryOp(Xor, (float)((!!a) ^ (!!b)));
     DefBinaryOp(MaskNegative, b >= 0 ? a : 0);
+    DefBinaryOp(ElementwiseProductWithSigmoidDerivative, a * SigmoidDerivative(b));
 #pragma pop_macro("DefBinaryOp")
 
 #pragma push_macro("DefTernaryOp")
     #define DefTernaryOp(op, expr) template<class ElemType> DECL ElemType Op ## op(ElemType a, ElemType b, ElemType c) { return expr; }
 
-    DefTernaryOp(Cond, a ? b : c);
+    DefTernaryOp(Cond, a ? b : c); DefTernaryOp(Clip, a < b ? b : (a > c ? c : a));
 #pragma pop_macro("DefTernaryOp")
 
 }}}

Source/Math/TensorView.h

@@ -64,7 +64,6 @@ namespace Microsoft { namespace MSR { namespace CNTK {
         void    Add ## oper ## Of(               const TensorView & a, ElemType alpha = 1.0f) { DoUnaryOpOf(1.0f, a, alpha, ElementWiseOperator::op ## oper); }
 
         ForAllUnaryOps(DeclareUnaryTensorOp);
-        ForAllParameterizedUnaryOps(DeclareUnaryTensorOp);
 #pragma pop_macro("DeclareUnaryTensorOp")
 
 #pragma push_macro("DeclareBinaryTensorOp")