fp16 - brain script add to CNTK.core.bs (microsoft#3617)

Author: rpengms

Source/CNTK/BrainScript/CNTKCoreLib/CNTK.core.bs

@@ -412,7 +412,7 @@ CNTK2 = [
     // Changes: dims -> shape
     DynamicAxis(tag='') = new ComputationNode [ operation = 'DynamicAxis' ; /*plus the function args*/  ]
     # TODO: Is it a good idea to default to "feature"?
-    Input(shape, dynamicAxis='', tag='feature') = new ComputationNode [ operation = 'InputValue' ; shape = new TensorShape [ /*shape*/ ] ; isImage = false /*plus the function args*/ ]
+    Input(shape, dynamicAxis='', tag='feature', precision=precision) = new ComputationNode [ operation = 'InputValue' ; shape = new TensorShape [ /*shape*/ ] ; isImage = false /*plus the function args*/ ]
 
     // 2. Variables and constants
     // Changes: ParameterTensor -> _Parameter; "dims" -> "shape"
@@ -570,14 +570,14 @@ ParameterTensor {dims, learningRateMultiplier = 1.0, init = ''/*|uniform|fixedVa
 ConstantFromString(literal, tag='') = ParameterTensor((0)/*dim, will be inferred*/, initFromLiteral = literal, learningRateMultiplier = 0.0)
 # TODO: Deprecate ConstantFromString() in favor of Constant(array expression)
 DynamicAxis(tag='') = new ComputationNode [ operation = 'DynamicAxis' ; /*plus the function args*/  ]
-Input(dims, dynamicAxis='', sparse=false, tag='feature') =
-     if sparse then SparseInput(dims, dynamicAxis=dynamicAxis, tag=tag)
+Input(dims, dynamicAxis='', sparse=false, tag='feature', precision=precision) =
+     if sparse then SparseInput(dims, dynamicAxis=dynamicAxis, tag=tag, precision=precision)
      else new ComputationNode [ operation = 'InputValue' ; shape = new TensorShape [ /*dims*/ ] ; isImage = false /*plus the function args*/ ]
 # TODO: change from dynamicAxis by name to dynamicAxis being an actual object
 # the following variants of Input() are deprecated
-SparseInput(dims, dynamicAxis='', tag='feature') = new ComputationNode [ operation = 'SparseInputValue' ; shape = new TensorShape [ /*dims*/ ] ; isImage = false /*plus the function args*/ ]
-ImageInput(imageWidth, imageHeight, imageChannels, imageLayout='CHW', dynamicAxis='', tag='feature') = new ComputationNode [ operation = 'InputValue' ; isImage = true /*plus the function args*/ ]
-SparseImageInput(imageWidth, imageHeight, imageChannels, imageLayout='CHW', dynamicAxis='', tag='feature') = new ComputationNode [ operation = 'SparseInputValue' ; isImage = true /*plus the function args*/ ]
+SparseInput(dims, dynamicAxis='', tag='feature', precision=precision) = new ComputationNode [ operation = 'SparseInputValue' ; shape = new TensorShape [ /*dims*/ ] ; isImage = false /*plus the function args*/ ]
+ImageInput(imageWidth, imageHeight, imageChannels, imageLayout='CHW', dynamicAxis='', tag='feature', precision=precision) = new ComputationNode [ operation = 'InputValue' ; isImage = true /*plus the function args*/ ]
+SparseImageInput(imageWidth, imageHeight, imageChannels, imageLayout='CHW', dynamicAxis='', tag='feature', precision=precision) = new ComputationNode [ operation = 'SparseInputValue' ; isImage = true /*plus the function args*/ ]
 EnvironmentInput(propertyName, tag='') = new ComputationNode [ operation = 'EnvironmentInput' /*plus the function args*/ ]
 # TODO: make 'dims' the first parameter, think ConstantTensor<dims> (val)
 ConstantTensor(val, dims, tag='') = ParameterTensor(dims, learningRateMultiplier = 0, initValue = val)
@@ -706,6 +706,8 @@ TransposeTimes(leftMatrix, rightMatrix, tag='') = new ComputationNode [ operatio
 QuantizedTimes(leftMatrix, rightMatrix, bitSmoothingA=1, bitSmoothingB=1, outputRank=1, inferInputRankToMap=-1, tag='') = new ComputationNode [ operation = 'QuantizedTimes' ; inputs = _AsNodes (leftMatrix : rightMatrix) /*plus the function args*/ ]
 Where(cond, tag='') = new ComputationNode [ operation = 'Where' ; inputs = _AsNodes (cond) /*plus the function args*/ ]
 
+Cast(node, precision='', tag='') = new ComputationNode [ operation = 'Cast' ; inputs = _AsNodes (node) /*plus the function args*/ ]
+
 ##############################################################################
 # non-neural-network functions
 ##############################################################################

Source/ComputationNetworkLib/ComputationNetworkBuilder.cpp

@@ -208,12 +208,54 @@ shared_ptr<ComputationNodeBase> NewComputationNodeFromConfig(const Microsoft::MS
     wstring precision = configp->Get(L"precision"); // dispatch on ElemType
     wstring operationName = configp->Get(L"operation");
     ComputationNodeBasePtr node;
-    if (precision == L"float")
-        node = CreateNode<float>(operationName, configp);
-    else if (precision == L"double")
-        node = CreateNode<double>(operationName, configp);
+    if (operationName == OperationName2Of(CastNode))
+    {
+        auto inputs = ComputationNodeBase::GetInputsFromConfig(configp);
+        if (inputs.empty())
+            RuntimeError("NewComputationNodeFromConfig: No inputs found for Cast node.");
+
+        if (precision == L"float16" || precision == L"half")
+        {
+            if (inputs[0]->Is<ComputationNode<float>>())
+                node = CreateNode2<half, float>(operationName, configp);
+            else if (inputs[0]->Is<ComputationNode<double>>())
+                node = CreateNode2<half, double>(operationName, configp);
+            else
+                RuntimeError("NewComputationNodeFromConfig: for CastNode to cast to half, input must be  'float' or 'double'");
+        }
+        else if (precision == L"float")
+        {
+            if (inputs[0]->Is<ComputationNode<half>>())
+                node = CreateNode2<float, half>(operationName, configp);
+            else if (inputs[0]->Is<ComputationNode<double>>())
+                    node = CreateNode2<float, double>(operationName, configp);
+            else
+                RuntimeError("NewComputationNodeFromConfig: for CastNode to cast to float, input must be  'float16' or 'double'");
+        }
+        else if (precision == L"double")
+        {
+            if (inputs[0]->Is<ComputationNode<half>>())
+                node = CreateNode2<double, half>(operationName, configp);
+            else if (inputs[0]->Is<ComputationNode<float>>())
+                node = CreateNode2<double, float>(operationName, configp);
+            else
+                RuntimeError("NewComputationNodeFromConfig: for CastNode to cast to double, input must be  'float' or 'float16'");
+        }
+        else
+            RuntimeError("NewComputationNodeFromConfig: CastNode - need to specify 'precision' parameter: 'float', 'double' or 'float16'.");
+    }
     else
-        RuntimeError("NewStandardNode: Invalid value '%ls' for 'precision' parameter. Must be 'float' or 'double'.", precision.c_str());
+    {
+        if (precision == L"float")
+            node = CreateNode<float>(operationName, configp);
+        else if (precision == L"float16" || precision == L"half")
+            node = CreateNode<half>(operationName, configp);
+        else if (precision == L"double")
+            node = CreateNode<double>(operationName, configp);
+        else
+            RuntimeError("NewComputationNodeFromConfig: Invalid value '%ls' for 'precision' parameter. Must be 'float16', 'float' or 'double'.", precision.c_str());
+    }
+
     // add a tag
     // Tags are used to declare special node types to ComputationNetwork.
     // For now we support only a single tag, but we could in the future easily extend this to an array of tags.

Source/ComputationNetworkLib/LinearAlgebraNodes.h

@@ -2013,6 +2013,7 @@ class CastNode : public UnaryElementWiseNode<ElemType>
     static const std::wstring TypeName() { return L"Cast"; }
 
 public:
+    DeclareConstructorFromConfig(CastNode);
     CastNode(DEVICEID_TYPE deviceId, const wstring& name)
         : Base(deviceId, name)
     {