GradientScaleLayer.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using MyCaffe.basecode;
using MyCaffe.common;
using MyCaffe.param;
 
namespace MyCaffe.layers
{
    public class GradientScaleLayer<T> : NeuronLayer<T>
    {
        double m_dfLowerBound;
        double m_dfUpperBound;
        double m_dfAlpha;
        double m_dfMaxIter;
        double m_dfCoeff;
        Stopwatch m_swOutput = new Stopwatch();
 
        public GradientScaleLayer(CudaDnn<T> cuda, Log log, LayerParameter p)
            : base(cuda, log, p)
        {
            m_type = LayerParameter.LayerType.GRADIENTSCALER;
        }
 
        public override void LayerSetUp(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            base.LayerSetUp(colBottom, colTop);
 
            GetIterationArgs args = getCurrentIteration();
            if (args != null && args.CurrentPhase != Phase.TRAIN)
                return;
 
            m_log.CHECK(args != null, "WARNING: The OnGetIteration event is not connected!");
 
            m_dfLowerBound = m_param.gradient_scale_param.lower_bound;
            m_dfUpperBound = m_param.gradient_scale_param.upper_bound;
            m_dfAlpha = m_param.gradient_scale_param.alpha;
            m_dfMaxIter = m_param.gradient_scale_param.max_iter;
            m_dfCoeff = 1.0; // default adaptation coefficient.
 
            m_log.CHECK_LE(m_dfLowerBound, m_dfUpperBound, "The lower bound must be <= the upper bound.");
            m_log.CHECK_GE(m_dfAlpha, 0, "The alpha value must be >= 0.0");
            m_log.CHECK_GE(m_dfCoeff, 1, "The max_iter must be >= 1.0");
 
            int nIteration = (args == null) ? 1 : args.Iteration;
            double dfProgress = Math.Min(1.0, (double)nIteration / m_dfMaxIter);
            double dfHeight = m_dfUpperBound - m_dfLowerBound;
 
            m_dfCoeff = 2.0 * dfHeight / (1.0 + Math.Exp(-m_dfAlpha * dfProgress)) - dfHeight + m_dfLowerBound;
            m_log.WriteLine("iter = " + nIteration.ToString() + " progress = " + dfProgress.ToString() + " coeff = " + m_dfCoeff.ToString());
            m_swOutput.Start();
        }
 
        protected override void forward(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            colTop[0].ShareData(colBottom[0]);
        }
 
        protected override void backward(BlobCollection<T> colTop, List<bool> rgbPropagateDown, BlobCollection<T> colBottom)
        {
            if (!rgbPropagateDown[0])
                return;
 
            GetIterationArgs args = getCurrentIteration();
            if (args != null && args.CurrentPhase != Phase.TRAIN)
                return;
 
            int nCount = colTop[0].count();
            long hTopDiff = colTop[0].gpu_diff;
            long hBottomDiff = colBottom[0].mutable_gpu_diff;
 
            int nIteration = (args == null) ? 1 : args.Iteration;
            double dfProgress = Math.Min(1.0, (double)nIteration / m_dfMaxIter);
            double dfHeight = m_dfUpperBound - m_dfLowerBound;
 
            m_dfCoeff = 2.0 * dfHeight / (1.0 + Math.Exp(-m_dfAlpha * dfProgress)) - dfHeight + m_dfLowerBound;
 
            if (m_swOutput.Elapsed.TotalMilliseconds > 1000)
            {
                m_log.WriteLine("iter = " + nIteration.ToString() + " progress = " + dfProgress.ToString() + " coeff = " + m_dfCoeff.ToString());
                m_swOutput.Restart();
            }
 
            m_cuda.scale(nCount, -m_dfCoeff, hTopDiff, hBottomDiff);
        }
    }
}