merge fp16_bs fix (microsoft#3609)

fp16 brain script fix accumulator

Author: rpengms

Source/SGDLib/AccumulatorAggregation.h

@@ -95,7 +95,7 @@ template <typename ElemType>
 void UpdateEpochEvaluationForAccumulatedResult(
     std::vector<EpochCriterion>& epochEvalErrors,
     const std::vector<ComputationNodeBasePtr>& evaluationNodes,
-    CriterionAccumulator<ElemType> localEpochEvalErrors,
+    CriterionAccumulatorBase& localEpochEvalErrors,
     std::function<bool(ComputationNodeBasePtr)> containsAccumulatedResult
     )
 {
@@ -120,7 +120,7 @@ void AggregateAccumulatorValuesAndUpdateEpochEvaluation(
     std::shared_ptr<MPIWrapper> mpi,
     std::vector<EpochCriterion>& epochEvalErrors,
     const std::vector<ComputationNodeBasePtr>& evaluationNodes,
-    CriterionAccumulator<ElemType> localEpochEvalErrors,
+    CriterionAccumulatorBase& localEpochEvalErrors,
     std::function<bool(ComputationNodeBasePtr)> containsAccumulatedResult,
     size_t packThresholdSizeInBytes = DEFAULT_PACK_THRESHOLD_SIZE_IN_BYTES)
 {

Source/SGDLib/Criterion.h

@@ -74,8 +74,17 @@ struct EpochCriterion : public std::pair<double, size_t>
 
 // We accumulate criteria in this struct.
 // Criteria are accumulated together with their counts (counts depend on sequence lengths, and different criteria may have different sequence lengths).
+struct CriterionAccumulatorBase
+{
+    CriterionAccumulatorBase() {};
+    virtual ~CriterionAccumulatorBase() {};
+    virtual const CriterionAccumulatorBase& Add(size_t i, size_t numSamplesInMinibatch) = 0;
+    virtual const CriterionAccumulatorBase& Assign(size_t i, size_t numSamplesInMinibatch) = 0;
+    virtual EpochCriterion GetCriterion(size_t i) const = 0;
+};
+
 template <class ElemType>
-struct CriterionAccumulator
+struct CriterionAccumulator : CriterionAccumulatorBase
 {
     // constructor params:
     //  criterionNodes                  - list of criterion nodes
@@ -91,16 +100,16 @@ struct CriterionAccumulator
     }
     // 'i' is the index of the element we add into (multiple eval criteria share the same matrix object)
     // Use 'reset=true' to not accumulate but overwrite.
-    const CriterionAccumulator& Add(size_t i, size_t numSamplesInMinibatch)
+    virtual const CriterionAccumulator& Add(size_t i, size_t numSamplesInMinibatch) override
     {
         return Accumulate(i, numSamplesInMinibatch, /*reset=*/false);
     }
-    const CriterionAccumulator& Assign(size_t i, size_t numSamplesInMinibatch)
+    virtual const CriterionAccumulator& Assign(size_t i, size_t numSamplesInMinibatch) override
     {
         return Accumulate(i, numSamplesInMinibatch, /*reset=*/true);
     }
     // retrieve an accumulated result as a pair (numerator, denominator)
-    EpochCriterion GetCriterion(size_t i) const
+    virtual EpochCriterion GetCriterion(size_t i) const override
     {
         // BUGBUG: For unknown reasons, this (or the other below) check makes a difference for MPI configs.
         //         If it is left out, then training and test configs end up being scaled by the same factor close to 1.
@@ -194,4 +203,26 @@ struct CriterionAccumulator
     const std::vector<ComputationNodeBasePtr> m_accumulatorCriterionNodes;
 };
 
+class CriterionAccumulatorFactory
+{
+public:
+    template <class ElemType>
+    static shared_ptr<CriterionAccumulatorBase> CreateCriterionAccumulator(
+        const std::vector<ComputationNodeBasePtr>& criterionNodes, DEVICEID_TYPE deviceId,
+        const std::vector<ComputationNodeBasePtr>& accumulatorCriterionNodesNodes = {})
+    {
+        // Both half and float use float as accumulator
+        if (std::is_same<ElemType, float>() || std::is_same<ElemType, half>())
+        {
+            return make_shared<CriterionAccumulator<float>>(criterionNodes, deviceId, accumulatorCriterionNodesNodes);
+        }
+        else if (std::is_same<ElemType, double>())
+        {
+            return make_shared<CriterionAccumulator<double>>(criterionNodes, deviceId, accumulatorCriterionNodesNodes);
+        }
+        RuntimeError("CreateCriterionAccumulator: unsupported node element type!");
+    }
+
+};
+
 }}}

Source/SGDLib/SGD.cpp

@@ -1148,10 +1148,14 @@ size_t SGD<ElemType>::TrainOneEpoch(ComputationNetworkPtr net,
     // NOTE: the following two local matrices are not used in distGradAgg path
     // assume only one training criterion node for each epoch.
     // The criterion values are accumulated here over the minibatches (without having to pull them off the GPU).
-    CriterionAccumulator<ElemType> localEpochCriterion(criterionNodes, net->GetDeviceId());
-    CriterionAccumulator<ElemType> localEpochEvalErrors(
+    // For half, the cr and error nodes should be float nodes
+    shared_ptr<CriterionAccumulatorBase> localEpochCriterionPtr = CriterionAccumulatorFactory::CreateCriterionAccumulator<ElemType>(
+        criterionNodes, net->GetDeviceId());
+    shared_ptr<CriterionAccumulatorBase> localEpochEvalErrorsPtr = CriterionAccumulatorFactory::CreateCriterionAccumulator<ElemType>(
         evaluationNodes, net->GetDeviceId(),
         {evaluationNodesWhichAccumulateResult.begin(), evaluationNodesWhichAccumulateResult.end()});
+    CriterionAccumulatorBase& localEpochCriterion = *localEpochCriterionPtr;
+    CriterionAccumulatorBase& localEpochEvalErrors = *localEpochEvalErrorsPtr;
 
     // --- MAIN MINIBATCH LOOP
 

Source/SGDLib/SimpleEvaluator.h

@@ -108,9 +108,10 @@ class SimpleEvaluator
         if (numSubminibatchesNeeded > 1)
             smbDispatcher.Init(m_net, learnableNodes, criterionNodes, evalNodes);
 
-        CriterionAccumulator<ElemType> localEpochEvalErrors(
+        shared_ptr<CriterionAccumulatorBase> localEpochEvalErrorsPtr = CriterionAccumulatorFactory::CreateCriterionAccumulator<ElemType>(
             evalNodes, m_net->GetDeviceId(),
             {evalNodesWhichAccumulateResult.begin(), evalNodesWhichAccumulateResult.end()});
+        CriterionAccumulatorBase& localEpochEvalErrors = *localEpochEvalErrorsPtr;
 
         const size_t numIterationsBeforePrintingProgress = 100;
         size_t numItersSinceLastPrintOfProgress = 0;