Layer.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Drawing;
using System.Diagnostics;
using MyCaffe.basecode;
using MyCaffe.db.image;
using MyCaffe.common;
using MyCaffe.param;
using System.IO;
using System.Reflection;
 
namespace MyCaffe.layers
{
    public abstract class Layer<T> : IDisposable
    {
        protected LayerParameter.LayerType m_type = LayerParameter.LayerType._MAX;
        protected CudaDnn<T> m_cuda;
        protected Log m_log;
        protected LayerParameter m_param;
        protected Phase m_phase;
        protected BlobCollection<T> m_colBlobs;
        protected BlobCollection<T> m_colInternalBlobs = new BlobCollection<T>();
        protected DictionaryMap<bool> m_rgbParamPropagateDown;
        protected DictionaryMap<double> m_rgLoss;
        protected T m_tOne;
        protected T m_tZero;
        protected bool m_bEnablePassthrough = false;
        protected bool m_bUseHalfSize = false;
        protected bool m_bConvertTopOnFwd = false;
        protected bool m_bConvertTopOnBwd = true;
        protected bool m_bConvertBottom = true;
        protected bool m_bReshapeOnForwardNeeded = true;
        protected bool m_bNetReshapeRequest = false;
        protected LayerParameter.LayerType? m_parentLayerType = null;
 
        private List<List<int>> m_rgrgLastBottomShape = new List<List<int>>();
        private List<List<int>> m_rgrgLastTopShape = new List<List<int>>();
 
        private double m_dfForwardTiming = 0;
        private double m_dfForwardAverageTiming = 0;
        private double m_dfBackwardTiming = 0;
        private double m_dfBackwardAverageTiming = 0;
        private double m_dfAverageInterval = 20.0;
        private Stopwatch m_swTiming = new Stopwatch();
        private WorkspaceArgs m_argsWs = new WorkspaceArgs(0, 0);
 
        public event EventHandler<WorkspaceArgs> OnGetWorkspace;
        public event EventHandler<WorkspaceArgs> OnSetWorkspace;
        public event EventHandler<GetIterationArgs> OnGetIteration;
        public event EventHandler<GetWorkBlobArgs<T>> OnDebug;
 
        enum CONVERT_TYPE
        {
            BASE_TO_HALF,
            HALF_TO_BASE
        }
 
        public Layer(CudaDnn<T> cuda, Log log, LayerParameter p)
        {
            m_cuda = cuda;
            m_log = log;
            m_param = p.Clone(true);
            m_phase = p.phase;
            m_rgbParamPropagateDown = new DictionaryMap<bool>(false);
            m_rgLoss = new DictionaryMap<double>(0.0);
            m_colBlobs = new BlobCollection<T>();
 
            for (int i = 0; i < p.blobs.Count; i++)
            {
                m_colBlobs.Add(new Blob<T>(cuda, log, p.blobs[i]));
            }
 
            m_tOne = (T)Convert.ChangeType(1, typeof(T));
            m_tZero = (T)Convert.ChangeType(0, typeof(T));            
        }
 
        public void Dispose()
        {
            dispose();
        }
 
        protected virtual void dispose()
        {
        }
        
        protected void dispose(ref Layer<T> l)
        {
            if (l != null)
            {
                l.Dispose();
                l = null;
            }
        }
 
        protected void dispose(ref Blob<T> b)
        {
            if (b != null)
            {
                b.Dispose();
                b = null;
            }
        }
 
        protected void dispose(ref BlobCollection<T> rg, bool bSetToNull = true)
        {
            if (rg != null)
            {
                rg.Dispose();
                if (bSetToNull)
                    rg = null;
            }
        }
 
        public virtual void ConnectLoss(LossLayer<T> layer)
        {
        }
 
        public LayerParameter.LayerType? parent_layer_type
        {
            get { return m_parentLayerType; }
        }
 
        public virtual bool SupportsPreProcessing
        {
            get { return false; }
        }
 
        public virtual bool SupportsPostProcessing
        {
            get { return false; }
        }
 
        public virtual bool SupportsPostProcessingLogits
        {
            get { return false; }
        }
 
        public virtual bool SupportsPostProcessingFullOutput
        {
            get { return false; }
        }
 
        public virtual BlobCollection<T> PreProcessInput(PropertySet customInput, out int nSeqLen, BlobCollection<T> colBottom = null)
        {
            nSeqLen = 0;
            return colBottom;
        }
 
        public virtual bool PreProcessInput(string strEncInput, int? nDecInput, BlobCollection<T> colBottom)
        {
            return false;
        }
 
        public virtual List<Tuple<string, int, double>> PostProcessOutput(Blob<T> blobSofmtax, int nK = 1)
        {
            return null;
        }
        public virtual List<Tuple<string, int, double>> PostProcessLogitsOutput(int nCurIdx, Blob<T> blobLogits, Layer<T> softmax, int nAxis, int nK = 1)
        {
            return null;
        }
        public virtual string PostProcessFullOutput(Blob<T> blobSoftmax)
        {
            return null;
        }
 
        public virtual string PostProcessOutput(int nIdx)
        {
            return null;
        }
 
        public virtual void SetOnDebug(EventHandler<GetWorkBlobArgs<T>> fn)
        {
            OnDebug += fn;
        }
 
        public virtual void ResetOnDebug(EventHandler<GetWorkBlobArgs<T>> fn)
        {
            OnDebug -= fn;
        }
 
        public virtual bool ReInitializeParameters(WEIGHT_TARGET target)
        {
            return true;
        }
 
        protected GetIterationArgs getCurrentIteration()
        {
            GetIterationArgs args = null;
 
            if (OnGetIteration != null)
            {
                args = new GetIterationArgs();
                OnGetIteration(this, args);               
            }
 
            return args;
        }
 
        public void SetNetReshapeRequest()
        {
            m_bNetReshapeRequest = true;
        }
 
        public void SetPhase(Phase phase)
        {
            m_phase = phase;
            m_param.phase = phase;
        }
 
        public void Setup(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            try
            {
                m_rgrgLastBottomShape = new List<List<int>>();
                m_rgrgLastTopShape = new List<List<int>>();
 
                CheckBlobCounts(colBottom, colTop);
                setup_internal_blobs(m_colInternalBlobs);
                LayerSetUp(colBottom, colTop);
                Reshape(colBottom, colTop);
                setShapes(colBottom, colTop);
                SetLossWeights(colTop);
 
                if (colBottom != colTop)
                {
                    for (int i = 0; i < colBottom.Count; i++)
                    {
                        if (i < colTop.Count && colBottom[i] != colTop[i])
                        {
                            colTop[i].CopyParameters(colBottom[i]);
                        }
                    }
                }
            }
            catch (Exception excpt)
            {
                if (m_param != null)
                    throw new Exception("Layer: '" + m_param.name + "' (" + m_param.type.ToString() + ") Error: " + excpt.Message, excpt);
                else
                    throw excpt;
            }
        }
 
        public abstract void LayerSetUp(BlobCollection<T> colBottom, BlobCollection<T> colTop);
 
        public virtual void SetNetParameterUsed(NetParameter np)
        {
        }
 
        public abstract void Reshape(BlobCollection<T> colBottom, BlobCollection<T> colTop);
 
        protected long convert_to_full(int nCount, long hMem)
        {
            if (OnGetWorkspace == null || OnSetWorkspace == null)
                throw new Exception("The OnGetWorkSpace and OnSetWorkspace events must be connected!");
 
            ulong lSize = (ulong)nCount * CudaDnn<T>.basetype_size(false);
            WorkspaceArgs args = getWorkspace();
            if (args.WorkspaceSizeInBytes < lSize)
            {
                setWorkspace(lSize);
                args = getWorkspace();
            }
 
            m_cuda.copy(nCount, hMem, args.WorkspaceData, 0, 0, -1, null, false);
            return args.WorkspaceData;
        }
 
        protected void convert(BlobCollection<T> col)
        {
            ulong lMaxSize = 0;
            bool bConversionNeeded = false;
 
            foreach (Blob<T> b in col)
            {
                if ((m_bUseHalfSize && !b.HalfSize) ||
                    (!m_bUseHalfSize && b.HalfSize))
                {
                    ulong lSize = b.GetConversionWorkSize(m_bUseHalfSize);
                    if (lMaxSize < lSize)
                        lMaxSize = lSize;
 
                    bConversionNeeded = true;
                }
            }
 
            if (!bConversionNeeded)
                return;
 
            if (OnGetWorkspace == null || OnSetWorkspace == null)
                throw new Exception("The OnGetWorkSpace and OnSetWorkspace events must be connected!");
 
            WorkspaceArgs args = getWorkspace();
            if (args.WorkspaceSizeInBytes < lMaxSize)
            {
                setWorkspace(lMaxSize);
                args = getWorkspace();
            }
 
            foreach (Blob<T> b in col)
            {
                if (m_bUseHalfSize && !b.HalfSize)
                    b.ConvertToHalf(args.WorkspaceData, args.WorkspaceSizeInBytes, true, true);
                else if (!m_bUseHalfSize && b.HalfSize)
                    b.ConvertToBase(args.WorkspaceData, args.WorkspaceSizeInBytes, true, true);
            }
        }
 
        public void ConvertToBase(BlobCollection<T> col)
        {
            if (OnGetWorkspace == null || OnSetWorkspace == null)
                return;
 
            ulong lMaxSize = 0;
            bool bConversionNeeded = false;
 
            foreach (Blob<T> b in col)
            {
                if (b.HalfSize)
                {
                    ulong lSize = b.GetConversionWorkSize(false);
                    if (lMaxSize < lSize)
                        lMaxSize = lSize;
 
                    bConversionNeeded = true;
                }
            }
 
            if (!bConversionNeeded)
                return;
 
            WorkspaceArgs args = getWorkspace();
            if (args.WorkspaceSizeInBytes < lMaxSize)
            {
                setWorkspace(lMaxSize);
                args = getWorkspace();
            }
 
            foreach (Blob<T> b in col)
            {
                b.ConvertToBase(args.WorkspaceData, args.WorkspaceSizeInBytes, true, true);
            }
        }
 
        protected virtual bool reshapeNeeded(BlobCollection<T> colBottom, BlobCollection<T> colTop, bool bReset = true)
        {
            return true;
 
            // Memory optimizations require reshaping now on each pass.
            //if (!bReset)
            //    return m_bReshapeOnForwardNeeded;
 
            //if (!compareShapes(colBottom, colTop))
            //{
            //    m_bReshapeOnForwardNeeded = true;
            //    return true;
            //}
            //else
            //{
            //    m_bReshapeOnForwardNeeded = false;
            //    return false;
            //}
        }
 
        protected bool compareShapes(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            if (!compareShapes(colBottom, m_rgrgLastBottomShape))
                return false;
 
            if (!compareShapes(colTop, m_rgrgLastTopShape))
                return false;
 
            return true;
        }
 
        private bool compareShapes(BlobCollection<T> col, List<List<int>> rgrg)
        {
            if (rgrg.Count != col.Count)
                return false;
 
            for (int i = 0; i < col.Count; i++)
            {
                int nCount = col[i].shape().Count;
                if (rgrg[i].Count != nCount)
                    return false;
 
                for (int j = 0; j < nCount; j++)
                {
                    if (col[i].shape()[j] != rgrg[i][j])
                        return false;
                }
            }
 
            return true;
        }
 
        protected void setShapes(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            setShapes(colBottom, ref m_rgrgLastBottomShape);
            setShapes(colTop, ref m_rgrgLastTopShape);
        }
 
        private void setShapes(BlobCollection<T> col, ref List<List<int>> rgrg)
        {
            if (rgrg.Count != col.Count)
                rgrg = new List<List<int>>(col.Count);
 
            for (int i = 0; i < col.Count; i++)
            {
                int nCount = col[i].shape().Count;
                if (rgrg.Count < col.Count)
                    rgrg.Add(new List<int>());
                else if (rgrg[i].Count != nCount)
                    rgrg[i] = new List<int>(nCount);
 
                for (int j = 0; j < nCount; j++)
                {
                    nCount = col[i].shape().Count;
                    if (rgrg[i].Count < nCount)
                        rgrg[i].Add(col[i].shape()[j]);
                    else
                        rgrg[i][j] = col[i].shape()[j];
                }
            }
        }
 
        public double Forward(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            try
            {
                m_swTiming.Restart();
                double dfLoss = 0;
 
                if (m_bConvertBottom)
                    convert(colBottom);
 
                Reshape(colBottom, colTop);
 
                if (m_bConvertTopOnFwd)
                    convert(colTop);
 
                forward(colBottom, colTop);
 
                for (int i = 0; i < colTop.Count; i++)
                {
                    if (loss(i) == 0)
                        continue;
 
                    int nCount = colTop[i].count();
                    long hData = colTop[i].gpu_data;
                    long hDiff = colTop[i].gpu_diff;
                    double dfBlobLoss = m_cuda.dot_double(nCount, hData, hDiff);
 
                    dfLoss += dfBlobLoss;
                }
 
                m_swTiming.Stop();
                m_dfForwardTiming = m_swTiming.Elapsed.TotalMilliseconds;
                m_dfForwardAverageTiming = getAveTiming(m_dfAverageInterval, m_dfForwardTiming, m_dfForwardAverageTiming);
 
                if (OnDebug != null)
                {
                    GetWorkBlobArgs<T> args = new GetWorkBlobArgs<T>();
                    OnDebug(this, args);
 
                    foreach (Blob<T> b in colTop)
                    {
                        Tuple<double, double, double, double> mm_data = b.minmax_data(args.Blob, true);
                        Tuple<double, double, double, double> mm_diff = b.minmax_diff(args.Blob, true);
 
                        if (mm_data.Item3 > 0 || mm_data.Item4 > 0)
                            throw new Exception("NAN or INF detected in the TOP '" + b.Name + "' Data for layer '" + m_param.name + "' on the forward pass.");
 
                        if (mm_diff.Item3 > 0 || mm_diff.Item4 > 0)
                            throw new Exception("NAN or INF detected in TOP '" + b.Name + "' Diff for layer '" + m_param.name + "' on the forward pass.");
                    }
                }
 
                return dfLoss;
            }
            catch (Exception excpt)
            {
                if (m_param != null)
                    throw new Exception("Layer: '" + m_param.name + "' (" + m_param.type.ToString() + ") Error: " + excpt.Message, excpt);
                else
                    throw excpt;
            }
        }
 
        protected abstract void forward(BlobCollection<T> colBottom, BlobCollection<T> colTop);
 
        public void Backward(BlobCollection<T> colTop, List<bool> rgbPropagateDown, BlobCollection<T> colBottom)
        {
            try
            {
                m_swTiming.Restart();
 
                if (m_bConvertTopOnBwd)
                    convert(colTop);
 
                convert(colBottom);
 
                backward(colTop, rgbPropagateDown, colBottom);
                m_swTiming.Stop();
                m_dfBackwardTiming = m_swTiming.Elapsed.TotalMilliseconds;
                m_dfBackwardAverageTiming = getAveTiming(m_dfAverageInterval, m_dfBackwardTiming, m_dfBackwardAverageTiming);
 
                if (OnDebug != null)
                {
                    GetWorkBlobArgs<T> args = new GetWorkBlobArgs<T>();
                    OnDebug(this, args);
 
                    foreach (Blob<T> b in colBottom)
                    {
                        Tuple<double, double, double, double> mm_data = b.minmax_data(args.Blob, true);
                        Tuple<double, double, double, double> mm_diff = b.minmax_diff(args.Blob, true);
 
                        if (mm_data.Item3 > 0 || mm_data.Item4 > 0)
                            throw new Exception("NAN or INF detected in the BOTTOM '" + b.Name + "' Data for layer '" + m_param.name + "' on the backward pass.");
 
                        if (mm_diff.Item3 > 0 || mm_diff.Item4 > 0)
                            throw new Exception("NAN or INF detected in the BOTTOM '" + b.Name + "' Diff for layer '" + m_param.name + "' on the backward pass.");
                    }
                }
            }
            catch (Exception excpt)
            {
                if (m_param != null)
                    throw new Exception("Layer: '" + m_param.name + "' (" + m_param.type.ToString() + ") Error: " + excpt.Message, excpt);
                else
                    throw excpt;
            }
        }
 
        protected abstract void backward(BlobCollection<T> colTop, List<bool> rgbPropagateDown, BlobCollection<T> colBottom);
 
        public BlobCollection<T> blobs
        {
            get { return m_colBlobs; }
        }
 
        public BlobCollection<T> internal_blobs
        {
            get { return m_colInternalBlobs; }
        }
 
        protected virtual void setup_internal_blobs(BlobCollection<T> col)
        {
        }
 
        public LayerParameter layer_param
        {
            get { return m_param; }
        }
 
        public double loss(int nTopIdx)
        {
            return m_rgLoss[nTopIdx];
        }
 
        public void set_loss(int nTopIdx, double dfLoss)
        {
            m_rgLoss[nTopIdx] = dfLoss;
        }
 
        public LayerParameter.LayerType type
        {
            get { return m_type; }
        }
 
        public virtual int ExactNumBottomBlobs
        {
            get { return -1; }
        }
 
        public virtual int MinBottomBlobs
        {
            get { return -1; }
        }
 
        public virtual int MaxBottomBlobs
        {
            get { return -1; }
        }
 
        public virtual int ExactNumTopBlobs
        {
            get { return -1; }
        }
 
        public virtual int MinTopBlobs
        {
            get { return -1; }
        }
 
        public virtual int MaxTopBlobs
        {
            get { return -1; }
        }
 
        public virtual bool EqualNumBottomTopBlobs
        {
            get { return false; }
        }
 
        public virtual bool AutoTopBlobs
        {
            get { return false; }
        }
 
        public virtual bool AllowForceBackward(int nBottomIdx)
        {
            return true;
        }
 
        public bool param_propagate_down(int nParamIdx)
        {
            return m_rgbParamPropagateDown[nParamIdx];
        }
 
        public void set_param_propagate_down(int nParamIdx, bool bPropagate)
        {
            m_rgbParamPropagateDown[nParamIdx] = bPropagate;
        }
 
        protected void CheckBlobCounts(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            if (ExactNumBottomBlobs >= 0)
                m_log.CHECK_EQ(ExactNumBottomBlobs, colBottom.Count, type.ToString() + " Layer takes " + ExactNumBottomBlobs.ToString() + " bottom blob(s) as input.");
 
            if (MinBottomBlobs >= 0)
                m_log.CHECK_LE(MinBottomBlobs, colBottom.Count, type.ToString() + " Layer takes at least " + MinBottomBlobs.ToString() + " bottom blob(s) as input.");
 
            if (MaxBottomBlobs >= 0)
                m_log.CHECK_GE(MaxBottomBlobs, colBottom.Count, type.ToString() + " Layer takes at most " + MaxBottomBlobs.ToString() + " bottom blob(s) as input.");
 
            if (ExactNumTopBlobs >= 0)
                m_log.CHECK_EQ(ExactNumTopBlobs, colTop.Count, type.ToString() + " Layer takes " + ExactNumTopBlobs.ToString() + " top blob(s) as output.");
 
            if (MinTopBlobs >= 0)
                m_log.CHECK_LE(MinTopBlobs, colTop.Count, type.ToString() + " Layer takes at least " + MinTopBlobs.ToString() + " top blob(s) as output.");
 
            if (MaxTopBlobs >= 0)
                m_log.CHECK_GE(MaxTopBlobs, colTop.Count, type.ToString() + " Layer takes at most " + MaxTopBlobs.ToString() + " top blob(s) as output.");
 
            if (EqualNumBottomTopBlobs)
                m_log.CHECK_EQ(colBottom.Count, colTop.Count, type.ToString() + " Layer produces one top blob as output for each bottom blob input.");
        }
 
        protected void SetLossWeights(BlobCollection<T> colTop)
        {
            if (m_param.loss_weight.Count > 0)
            {
                m_log.CHECK_EQ(colTop.Count, m_param.loss_weight.Count, "loss_weight must be unspecified or specified once per top blob.");
 
                for (int i = 0; i < colTop.Count; i++)
                {
                    double dfLossWeight = m_param.loss_weight[i];
 
                    if (dfLossWeight == 0)
                        continue;
 
                    set_loss(i, dfLossWeight);
                    colTop[i].SetDiff(dfLossWeight);
                }
            }
        }
 
        protected LayerParameter convertLayerParam(LayerParameter pChild, LayerParameter pParent)
        {
            if (pParent is LayerParameterEx<T>)
            {
                LayerParameterEx<T> pEx = pParent as LayerParameterEx<T>;
                return new LayerParameterEx<T>(pChild, pEx.SharedBlobs, pEx.SharedLayerBlobs, null);
            }
 
            return pChild;
        }
 
        protected bool shareParameter(Blob<T> b, List<int> rgMinShape, bool bAllowEndsWithComparison = false)
        {
            LayerParameterEx<T> paramEx = m_param as LayerParameterEx<T>;
            if (paramEx == null)
                return false;
 
            if (paramEx.SharedBlobs == null)
                return false;
 
            return paramEx.SharedBlobs.Share(b, rgMinShape, false, bAllowEndsWithComparison);
        }
 
        protected bool shareLayerBlob(Blob<T> b, List<int> rgMinShape)
        {
            LayerParameterEx<T> paramEx = m_param as LayerParameterEx<T>;
            if (paramEx == null)
                return false;
 
            if (paramEx.SharedLayerBlobs == null)
                return false;
 
            return paramEx.SharedLayerBlobs.Share(b, rgMinShape, false);
        }
 
        protected bool shareLayerBlobs(Layer<T> layer)
        {
            LayerParameterEx<T> paramEx = m_param as LayerParameterEx<T>;
            if (paramEx == null)
                return false;
 
            if (paramEx.SharedLayer == null)
                return false;
 
            if (paramEx.SharedLayer.blobs.Count != layer.blobs.Count)
                return false;
 
            for (int i = 0; i < paramEx.SharedLayer.blobs.Count; i++)
            {
                Blob<T> bSrc = paramEx.SharedLayer.blobs[i];
                Blob<T> bDst = layer.blobs[i];
 
                string strSrc = bSrc.shape_string;
                string strDst = bDst.shape_string;
                if (strSrc != strDst)
                {
                    m_log.WriteLine("WARNING: Cannot share blob '" + bSrc.Name + "'(" + strSrc + ") with blob '" + bDst.Name + "'(" + strDst + ") because the sizes differ!");
                    return false;
                }
 
                bSrc.Share(bDst);
            }
 
            if (paramEx.SharedLayer.internal_blobs.Count != layer.internal_blobs.Count)
                return false;
 
            for (int i = 0; i < paramEx.SharedLayer.internal_blobs.Count; i++)
            {
                Blob<T> bSrc = paramEx.SharedLayer.internal_blobs[i];
                Blob<T> bDst = layer.internal_blobs[i];
 
                string strSrc = bSrc.shape_string;
                string strDst = bDst.shape_string;
                if (strSrc != strDst)
                {
                    m_log.WriteLine("WARNING: Cannot share internal blob '" + bSrc.Name + "'(" + strSrc + ") with internal blob '" + bDst.Name + "'(" + strDst + ") because the sizes differ!");
                    return false;
                }
 
                bSrc.Share(bDst);
            }
 
            return true;
        }
 
        public double forward_timing
        {
            get { return m_dfForwardTiming; }
        }
 
        public double forward_timing_average
        {
            get { return m_dfForwardAverageTiming; }
        }
 
        public double backward_timing
        {
            get { return m_dfBackwardTiming; }
        }
 
        public double backward_timing_average
        {
            get { return m_dfBackwardAverageTiming; }
        }
 
        public void SetEnablePassthrough(bool bEnable)
        {
            m_bEnablePassthrough = bEnable;
        }
 
        protected virtual WorkspaceArgs getWorkspace()
        {
            if (OnGetWorkspace == null)
                return null;
 
            OnGetWorkspace(this, m_argsWs);
 
            return m_argsWs;
        }
 
        protected virtual bool setWorkspace(ulong lSizeInBytes)
        {
            if (OnSetWorkspace == null)
                return false;
 
            OnSetWorkspace(this, new WorkspaceArgs(0, lSizeInBytes));
            return true;
        }
 
        protected void check_nan(Blob<T> b)
        {
            double[] rg = convertD(b.update_cpu_data());
 
            for (int i = 0; i < rg.Length; i++)
            {
                if (double.IsNaN(rg[i]))
                    throw new Exception("NAN FOUND!");
            }
        }
 
        protected T convert(double df)
        {
            return (T)Convert.ChangeType(df, typeof(T));
        }
 
        protected T convert(float f)
        {
            return (T)Convert.ChangeType(f, typeof(T));
        }
 
        protected double convertD(T df)
        {
            return (double)Convert.ChangeType(df, typeof(double));
        }
 
        protected float convertF(T df)
        {
            return (float)Convert.ChangeType(df, typeof(float));
        }
 
        protected double[] convertD(T[] rg)
        {
            if (typeof(T) == typeof(double))
                return (double[])Convert.ChangeType(rg, typeof(double[]));
 
            double[] rgdf = new double[rg.Length];
            Array.Copy(rg, rgdf, rg.Length);
 
            return rgdf;
        }
 
        protected T[] convert(double[] rg)
        {
            if (typeof(T) == typeof(double))
                return (T[])Convert.ChangeType(rg, typeof(T[]));
 
            float[] rgf = new float[rg.Length];
            Array.Copy(Array.ConvertAll(rg, p => Convert.ToSingle(p)), rgf, rg.Length);
 
            return (T[])Convert.ChangeType(rgf, typeof(T[]));
        }
 
        protected float[] convertF(T[] rg)
        {
            if (typeof(T) == typeof(float))
                return (float[])Convert.ChangeType(rg, typeof(float[]));
 
            float[] rgf = new float[rg.Length];
            Array.Copy(Array.ConvertAll(rg, p => Convert.ToSingle(p)), rgf, rg.Length);
 
            return rgf;
        }
 
        protected T[] convert(float[] rg)
        {
            if (typeof(T) == typeof(float))
                return (T[])Convert.ChangeType(rg, typeof(T[]));
 
            T[] rgt = new T[rg.Length];
            Array.Copy(rg, rgt, rg.Length);
 
            return rgt;
        }
 
        protected int val_at(T[] rg, int nIdx)
        {
            return (int)Convert.ChangeType(rg[nIdx], typeof(int));
        }
 
        protected Size size_at(Blob<T> b)
        {
            T[] rg = b.update_cpu_data();
            int nHeight = val_at(rg, 0);
            int nWidth = (rg.Length > 1) ? val_at(rg, 1) : nHeight;
            return new Size(nWidth, nHeight);
        }
 
        private double getAveTiming(double dfInterval, double dfTiming, double dfAveTiming)
        {
            double dfRatio = 1.0 / m_dfAverageInterval;
            return (dfAveTiming * (1.0 - dfRatio)) + (dfTiming * dfRatio);
        }
 
        public static Layer<T> Create(CudaDnn<T> cuda, Log log, LayerParameter p, CancelEvent evtCancel, IXDatabaseBase db = null, TransferInput trxinput = null)
        {
            switch (p.type)
            {
                case LayerParameter.LayerType.ABSVAL:
                    return new AbsValLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.ACCURACY:
                    return new AccuracyLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.ARGMAX:
                    return new ArgMaxLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.BATCHNORM:
                    return new BatchNormLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.BATCHREINDEX:
                    return new BatchReindexLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.BNLL:
                    return new BNLLLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.BIAS:
                    return new BiasLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.CLIP:
                    return new ClipLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.CONCAT:
                    return new ConcatLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.CONSTANT:
                    return new ConstantLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.CONTRASTIVE_LOSS:
                    return new ContrastiveLossLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.CONVOLUTION:
                    return new ConvolutionLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.CROP:
                    return new CropLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.DECONVOLUTION:
                    return new DeconvolutionLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.IM2COL:
                    return new Im2colLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.DATA:
                    return new DataLayer<T>(cuda, log, p, db, evtCancel);
 
                case LayerParameter.LayerType.DATA_NORMALIZER:
                    return new DataNormalizerLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.DEBUG:
                    return new DebugLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.DROPOUT:
                    return new DropoutLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.DUMMYDATA:
                    return new DummyDataLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.ELTWISE:
                    return new EltwiseLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.ELU:
                    return new ELULayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.EMBED:
                    return new EmbedLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.EUCLIDEAN_LOSS:
                    return new EuclideanLossLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.EXP:
                    return new ExpLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.FILTER:
                    return new FilterLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.FLATTEN:
                    return new FlattenLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.GRADIENTSCALER:
                    return new GradientScaleLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.HINGE_LOSS:
                    return new HingeLossLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.IMAGE_DATA:
                    return new ImageDataLayer<T>(cuda, log, p, evtCancel);
 
                case LayerParameter.LayerType.INFOGAIN_LOSS:
                    return new InfogainLossLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.INNERPRODUCT:
                    return new InnerProductLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.INPUT:
                    return new InputLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.LOG:
                    return new LogLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.LABELMAPPING:
                    return new LabelMappingLayer<T>(cuda, log, p, db);
 
                case LayerParameter.LayerType.LRN:
                    return new LRNLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.MATH:
                    return new MathLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.MEMORYDATA:
                    return new MemoryDataLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.MEMORY_LOSS:
                    return new MemoryLossLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.MULTINOMIALLOGISTIC_LOSS:
                    return new MultinomialLogisticLossLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.MVN:
                    return new MVNLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.PARAMETER:
                    return new ParameterLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.POOLING:
                    return new PoolingLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.POWER:
                    return new PowerLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.PRELU:
                    return new PReLULayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.REDUCTION:
                    return new ReductionLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.RELU:
                    return new ReLULayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.RESHAPE:
                    return new ReshapeLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SCALE:
                    return new ScaleLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SIGMOID:
                    return new SigmoidLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SIGMOIDCROSSENTROPY_LOSS:
                    return new SigmoidCrossEntropyLossLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SOFTMAXCROSSENTROPY_LOSS:
                    return new SoftmaxCrossEntropyLossLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SOFTMAXCROSSENTROPY2_LOSS:
                    return new SoftmaxCrossEntropy2LossLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SILENCE:
                    return new SilenceLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SLICE:
                    return new SliceLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SOFTMAX:
                    return new SoftmaxLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SOFTMAXWITH_LOSS:
                    return new SoftmaxLossLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SPLIT:
                    return new SplitLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SPP:
                    return new SPPLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.SWISH:
                    return new SwishLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.TANH:
                    return new TanhLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.THRESHOLD:
                    return new ThresholdLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.TILE:
                    return new TileLayer<T>(cuda, log, p);
 
                // DEPRECIATED
                case LayerParameter.LayerType.LSTM_SIMPLE:
                    return new LSTMSimpleLayer<T>(cuda, log, p);
 
                case LayerParameter.LayerType.RNN:
                    return new RNNLayer<T>(cuda, log, p, evtCancel);
 
                case LayerParameter.LayerType.LSTM:
                    return new LSTMLayer<T>(cuda, log, p, evtCancel);
 
                case LayerParameter.LayerType.LSTM_UNIT:
                    return new LSTMUnitLayer<T>(cuda, log, p);
 
                default:
                    Layer<T> layer = createDynamicLayer(cuda, log, p, db, evtCancel);
                    if (layer != null)
                        return layer;
 
                    log.FAIL("Unknown layer type: " + p.type.ToString());
                    break;
            }
 
            throw new NotImplementedException("The layer type: " + p.type.ToString() + " is not implemented yet.");
        }
 
        private static Layer<T> createDynamicLayer(CudaDnn<T> cuda, Log log, LayerParameter p, IXDatabaseBase db, CancelEvent evtCancel)
        {
            string strDir = System.IO.Path.GetDirectoryName(new System.Uri(System.Reflection.Assembly.GetExecutingAssembly().CodeBase).LocalPath);
            string[] rgstrFiles = Directory.GetFiles(strDir, "mycaffe.layers.*.dll");
 
            foreach (string strFile in rgstrFiles)
            {
                FileInfo fi = new FileInfo(strFile);
                if (fi.Name.ToLower().IndexOf("mycaffe.layers.") == 0 && fi.Extension.ToLower() == ".dll")
                {
                    ILayerCreator icreator = loadCreator(strFile);
                    if (icreator != null)
                    {
                        Layer<T> layer;
 
                        if (typeof(T) == typeof(double))
                            layer = icreator.CreateDouble(cuda as CudaDnn<double>, log, p, evtCancel, db) as Layer<T>;
                        else
                            layer = icreator.CreateSingle(cuda as CudaDnn<float>, log, p, evtCancel, db) as Layer<T>;
 
                        if (layer != null)
                            return layer;
                    }
                }
            }
 
            return null;
        }
 
        private static ILayerCreator loadCreator(string strPath)
        {
            try
            {
                Assembly a = Assembly.LoadFile(strPath);
 
                foreach (Type t in a.GetTypes())
                {
                    if (t.IsPublic)
                    {
                        Type iface = t.GetInterface("ILayerCreator");
                        if (iface != null)
                        {
                            object obj = Activator.CreateInstance(t);
                            return (ILayerCreator)obj;
                        }
                    }
                }
 
                return null;
            }
            catch (Exception)
            {
                return null;
            }
        }
    }
 
    public class LayerParameterEx<T> : LayerParameter
    {
        BlobCollection<T> m_colSharedBlobs = null;
        BlobCollection<T> m_colLayerBlobs = new BlobCollection<T>();
        Layer<T> m_layer;
 
        public LayerParameterEx(LayerParameter p, BlobCollection<T> colBlobs, BlobCollection<T> colLayerBlobs, Layer<T> sharedLayer)
            : base(p)
        {
            m_colSharedBlobs = colBlobs;
            m_colLayerBlobs = colLayerBlobs;
            m_layer = sharedLayer;
        }
 
        public Layer<T> SharedLayer
        {
            get { return m_layer; }
        }
 
        public BlobCollection<T> SharedBlobs
        {
            get { return m_colSharedBlobs; }
        }
 
        public BlobCollection<T> SharedLayerBlobs
        {
            get { return m_colLayerBlobs; }
        }
 
        public override LayerParameter Clone(bool bCloneBlobs)
        {
            return new LayerParameterEx<T>(base.Clone(bCloneBlobs), m_colSharedBlobs, m_colLayerBlobs, m_layer);
        }
    }
}