BaseConvolutionLayer.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using MyCaffe.basecode;
using MyCaffe.param;
using MyCaffe.fillers;
using MyCaffe.common;
 
namespace MyCaffe.layers
{
    public abstract class BaseConvolutionLayer<T> : Layer<T> 
    {
        protected Blob<T> m_blobKernelShape;
        protected Blob<T> m_blobStride;
        protected Blob<T> m_blobPad;
        protected Blob<T> m_blobDilation;
        protected Blob<T> m_blobConvInputShape;
        protected List<int> m_rgColBufferShape;
        protected List<int> m_rgOutputShape = new List<int>();
        protected List<int> m_rgBottomShape = new List<int>();
        protected int m_nNumSpatialAxes;
        protected int m_nBottomDim;
        protected int m_nTopDim;
        protected int m_nChannelAxis;
        protected int m_nNum;
        protected int m_nChannels;
        protected int m_nGroup;
        protected int m_nOutSpatialDim;
        protected int m_nWeightOffset;
        protected int m_nNumOutput;
        protected bool m_bBiasTerm;
        protected bool m_bIs1x1;
        protected bool m_bForceNDim2col;
 
        int m_nNumKernelsIm2col;
        int m_nNumKernelsCol2im;
        int m_nConvOutChannels;
        int m_nConvInChannels;
        int m_nConvOutSpatialDim;
        int m_nKernelDim;
        int m_nColOffset;
        int m_nOutputOffset;
 
        Blob<T> m_blobColBuffer;
        Blob<T> m_blobBiasMultiplier;
 
        long m_hWorkspaceData = 0;
        ulong m_lWorkspaceSize = 0;
        bool m_bWorkspaceOwner = false;
 
        
        public BaseConvolutionLayer(CudaDnn<T> cuda, Log log, LayerParameter p)
            : base(cuda, log, p)
        {
            m_blobKernelShape = new Blob<T>(cuda, log);
            m_blobKernelShape.Name = m_param.name + " kernel_shape";
 
            m_blobStride = new Blob<T>(cuda, log);
            m_blobStride.Name = m_param.name + " stride";
 
            m_blobPad = new Blob<T>(cuda, log);
            m_blobPad.Name = m_param.name + " pad";
 
            m_blobDilation = new Blob<T>(cuda, log);
            m_blobDilation.Name = m_param.name + " dilation";
 
            m_blobConvInputShape = new Blob<T>(cuda, log);
            m_blobConvInputShape.Name = m_param.name + " conv_input_shape";
 
            m_blobColBuffer = new Blob<T>(cuda, log);
            m_blobColBuffer.Name = m_param.name + " conv_col_buffer";
 
            m_blobBiasMultiplier = new Blob<T>(cuda, log);
            m_blobBiasMultiplier.Name = m_param.name + "conv_bias_mult";
        }
 
        protected override void dispose()
        {
            m_blobKernelShape.Dispose();
            m_blobStride.Dispose();
            m_blobPad.Dispose();
            m_blobDilation.Dispose();
            m_blobConvInputShape.Dispose();
 
            m_blobColBuffer.Dispose();
            m_blobBiasMultiplier.Dispose();
 
            if (m_bWorkspaceOwner && m_hWorkspaceData != 0)
            {
                m_cuda.DisableGhostMemory();
                m_cuda.FreeMemory(m_hWorkspaceData);
                m_cuda.ResetGhostMemory();
                m_hWorkspaceData = 0;
                m_bWorkspaceOwner = false;
            }
 
            base.dispose();
        }
 
        protected ulong getWorkspaceLimitInBytes(bool bUseTensorCores = false)
        {
            // Specify workspace limit for kernels directly until we have a 
            // planning strategy and a rewrite of Caffe's GPU memory management.
            // default = 1024 * 1024 * 16;
            ulong lWorkspaceLimitBytes = ulong.MaxValue;
            if (m_param.convolution_param.cudnn_workspace_limit >= 0)
                lWorkspaceLimitBytes = (ulong)m_param.convolution_param.cudnn_workspace_limit;
 
            if (lWorkspaceLimitBytes != ulong.MaxValue)
                lWorkspaceLimitBytes *= 16;
 
            // When using Half Size memory, let CUDA pick the fastest workspace size and algorithm.
            if (m_bUseHalfSize || bUseTensorCores)
                lWorkspaceLimitBytes = ulong.MaxValue;
 
            // BUG Work Around
            // With cuDNN 7.0.5 and above we are seeing memory overwrite errors (from CUDA)
            //  when using more than 1 group and the workspace.
            //  * also confirmed in cuDNN 7.1.4 and CUDA 9.2 on driver 397.64, 398.36
            if (m_nGroup > 1 && !m_param.convolution_param.cudnn_workspace_allow_on_groups)
                lWorkspaceLimitBytes = 0; // sets option to NO_WORKSPACE for Bwd Filter and Data
 
            return lWorkspaceLimitBytes;
        }
 
        protected override void setup_internal_blobs(BlobCollection<T> col)
        {
            if (col.Count > 0)
                return;
 
            if (!m_param.convolution_param.useCudnn(m_nNumSpatialAxes) || reverse_dimensions())
            {
                col.Add(m_blobColBuffer);
                col.Add(m_blobBiasMultiplier);
            }
        }
 
        protected override WorkspaceArgs getWorkspace()
        {
            WorkspaceArgs args = base.getWorkspace();
 
            if (args != null)
                return args;
 
            m_bWorkspaceOwner = true;
            return new common.WorkspaceArgs(m_hWorkspaceData, m_lWorkspaceSize);
        }
 
        protected override bool setWorkspace(ulong lSizeInBytes)
        {
            if (!m_bWorkspaceOwner && base.setWorkspace(lSizeInBytes))
                return true;
 
            m_bWorkspaceOwner = true;
 
            if (lSizeInBytes < m_lWorkspaceSize)
                return true;
 
            m_lWorkspaceSize = lSizeInBytes;
            m_cuda.DisableGhostMemory();
 
            if (m_hWorkspaceData != 0)
                m_cuda.FreeMemory(m_hWorkspaceData);
 
            if (m_lWorkspaceSize > 0)
                m_hWorkspaceData = m_cuda.AllocMemory((long)m_lWorkspaceSize);
 
            m_cuda.ResetGhostMemory();
 
            return true;
        }
 
        public override bool ReInitializeParameters(WEIGHT_TARGET target)
        {
            base.ReInitializeParameters(target);
 
            if (target == WEIGHT_TARGET.WEIGHTS || target == WEIGHT_TARGET.BOTH)
            {
                Filler<T> filler = Filler<T>.Create(m_cuda, m_log, m_param.convolution_param.weight_filler);
                filler.Fill(m_colBlobs[0]);
            }
 
            if (m_param.convolution_param.bias_term && m_colBlobs.Count > 1 && (target == WEIGHT_TARGET.BOTH || target == WEIGHT_TARGET.BIAS))
            {
                Filler<T> fillerBias = Filler<T>.Create(m_cuda, m_log, m_param.convolution_param.bias_filler);
                fillerBias.Fill(m_colBlobs[1]);
            }
 
            return true;
        }
 
        public override void LayerSetUp(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            if (!reshapeNeeded(colBottom, colTop))
                return;
 
            // Configure the kernel size, padding, stride and inputs.
            ConvolutionParameter p = m_param.convolution_param;
 
            m_bForceNDim2col = p.force_nd_im2col;
            m_nChannelAxis = colBottom[0].CanonicalAxisIndex(p.axis);
 
            int nFirstSpatialAxis = m_nChannelAxis + 1;
            int nNumAxes = colBottom[0].num_axes;
 
            m_nNumSpatialAxes = nNumAxes - nFirstSpatialAxis;
 
            m_log.CHECK_GE(m_nNumSpatialAxes, 0, "The number of spatial axes must be zero or greater.");
 
            List<int> rgBottomDimBlobShape = new List<int>() { m_nNumSpatialAxes + 1 };
            List<int> rgSpaitalDimBlobShape = new List<int>() { Math.Max(m_nNumSpatialAxes, 1) };
 
            // Setup filter kernel dimensions (blobKernelShape)
            m_blobKernelShape.Reshape(rgSpaitalDimBlobShape);
            T[] rgKernelShape = m_blobKernelShape.mutable_cpu_data;
 
            if (p.kernel_h.HasValue || p.kernel_w.HasValue)
            {
                m_log.CHECK_EQ(m_nNumSpatialAxes, 2, "kernel_h & kernel_w can only be used in 2D convolution.");
                m_log.CHECK_EQ(0, p.kernel_size.Count, "Either kernel_size or kernel_h/w should be specified; not both.");
                rgKernelShape[0] = (T)Convert.ChangeType(p.kernel_h.Value, typeof(T));
                rgKernelShape[1] = (T)Convert.ChangeType(p.kernel_w.Value, typeof(T));
            }
            else
            {
                int nNumKernelDims = p.kernel_size.Count;
                m_log.CHECK(nNumKernelDims == 1 || nNumKernelDims == m_nNumSpatialAxes, "Kernel size must be specified once, or once per spatial dimension (kernel_size specified " + nNumKernelDims.ToString() + " times; " + m_nNumSpatialAxes.ToString() + " spatial dims);");
 
                for (int i = 0; i < m_nNumSpatialAxes; i++)
                {
                    int nIdx = (nNumKernelDims == 1) ? 0 : i;
                    rgKernelShape[i] = (T)Convert.ChangeType(p.kernel_size[nIdx], typeof(T));
                }
            }
 
            for (int i = 0; i < m_nNumSpatialAxes; i++)
            {
                m_log.CHECK_GT((int)Convert.ChangeType(rgKernelShape[i], typeof(int)), 0, "Filter dimension must be non-zero.");
            }
 
            m_blobKernelShape.mutable_cpu_data = rgKernelShape;
 
 
            // Setup stride dimensions (blobStride)
            m_blobStride.Reshape(rgSpaitalDimBlobShape);
            T[] rgStrideData = m_blobStride.mutable_cpu_data;
 
            if (p.stride_h.HasValue || p.stride_w.HasValue)
            {
                m_log.CHECK_EQ(m_nNumSpatialAxes, 2, "stride_h & stride_w can only be used in 2D convolution.");
                m_log.CHECK_EQ(0, p.stride.Count, "Either stride_size or stride_h/w should be specified; not both.");
                rgStrideData[0] = (T)Convert.ChangeType(p.stride_h.Value, typeof(T));
                rgStrideData[1] = (T)Convert.ChangeType(p.stride_w.Value, typeof(T));
            }
            else
            {
                int nNumStrideDims = p.stride.Count;
                m_log.CHECK(nNumStrideDims == 0 || nNumStrideDims == 1 || nNumStrideDims == m_nNumSpatialAxes, "Stride size must be specified once, or once per spatial dimension (stride specified " + nNumStrideDims.ToString() + " times; " + m_nNumSpatialAxes.ToString() + " spatial dims);");
                int nDefaultStride = 1;
 
                for (int i = 0; i < m_nNumSpatialAxes; i++)
                {
                    if (nNumStrideDims == 0)
                    {
                        rgStrideData[i] = (T)Convert.ChangeType(nDefaultStride, typeof(T));
                    }
                    else
                    {
                        int nIdx = (nNumStrideDims == 1) ? 0 : i;
                        rgStrideData[i] = (T)Convert.ChangeType(p.stride[nIdx], typeof(T));
                    }
                    m_log.CHECK_GT((int)Convert.ChangeType(rgStrideData[i], typeof(int)), 0, "Stride dimension must be non-zero.");
                }
            }
 
            m_blobStride.mutable_cpu_data = rgStrideData;
 
 
            // Setup pad dimensions (blobPad)
            m_blobPad.Reshape(rgSpaitalDimBlobShape);
            T[] rgPadData = m_blobPad.mutable_cpu_data;
 
            if (p.pad_h.HasValue || p.pad_w.HasValue)
            {
                m_log.CHECK_EQ(m_nNumSpatialAxes, 2, "pad_h & pad_w can only be used in 2D convolution.");
                m_log.CHECK_EQ(0, p.pad.Count, "Either pad_size or pad_h/w should be specified; not both.");
                rgPadData[0] = (T)Convert.ChangeType(p.pad_h.Value, typeof(T));
                rgPadData[1] = (T)Convert.ChangeType(p.pad_w.Value, typeof(T));
            }
            else
            {
                int nNumPadDims = p.pad.Count;
                m_log.CHECK(nNumPadDims == 0 || nNumPadDims == 1 || nNumPadDims == m_nNumSpatialAxes, "Pad size must be specified once, or once per spatial dimension (pad specified " + nNumPadDims.ToString() + " times; " + m_nNumSpatialAxes.ToString() + " spatial dims);");
                int nDefaultPad = 0;
 
                for (int i = 0; i < m_nNumSpatialAxes; i++)
                {
                    if (nNumPadDims == 0)
                    {
                        rgPadData[i] = (T)Convert.ChangeType(nDefaultPad, typeof(T));
                    }
                    else
                    {
                        int nIdx = (nNumPadDims == 1) ? 0 : i;
                        rgPadData[i] = (T)Convert.ChangeType(p.pad[nIdx], typeof(T));
                    }
                }
            }
 
            m_blobPad.mutable_cpu_data = rgPadData;
 
 
            // Setup dilation dimensions (blobDilation)
            m_blobDilation.Reshape(rgSpaitalDimBlobShape);
            T[] rgDilationData = m_blobDilation.mutable_cpu_data;
            int nNumDilationDims = p.dilation.Count;
 
            m_log.CHECK(nNumDilationDims == 0 || nNumDilationDims == 1 || nNumDilationDims == m_nNumSpatialAxes, "Dilation size must be specified once, or once per spatial dimension (dilation specified " + nNumDilationDims.ToString() + " times; " + m_nNumSpatialAxes.ToString() + " spatial dims);");
            int nDefaultDilation = 1;
 
            for (int i = 0; i < m_nNumSpatialAxes; i++)
            {
                if (nNumDilationDims == 0)
                {
                    rgDilationData[i] = (T)Convert.ChangeType(nDefaultDilation, typeof(T));
                }
                else
                {
                    int nIdx = (nNumDilationDims == 1) ? 0 : i;
                    rgDilationData[i] = (T)Convert.ChangeType(p.dilation[nIdx], typeof(T));
                }
            }
 
            m_blobDilation.mutable_cpu_data = rgDilationData;
 
 
            // Special case: im2col is the identity for 1x1 convolution with stride 1
            // add no padding, so flag for skipping the buffer and transformation.
            m_bIs1x1 = true;
 
            for (int i = 0; i < m_nNumSpatialAxes; i++)
            {
                if (!(val_at(rgKernelShape, i) == 1 && 
                      val_at(rgStrideData, i) == 1 && 
                      val_at(rgPadData, i) == 0))
                {
                    m_bIs1x1 = false;
                    break;
                }
            }
 
            // Configure output channels and groups.
            m_nChannels = colBottom[0].shape(m_nChannelAxis);
            m_nNumOutput = (int)p.num_output;
            m_log.CHECK_GT(m_nNumOutput, 0, "Output count must be greater than zero.");
 
            m_nGroup = (int)p.group;
            m_log.CHECK_EQ(m_nChannels % m_nGroup, 0, "The channels must span evenly across the groups.");
            m_log.CHECK_EQ(m_nNumOutput % m_nGroup, 0, "The number of output should be a in multiples of group.");
 
            if (reverse_dimensions())
            {
                m_nConvOutChannels = m_nChannels;
                m_nConvInChannels = m_nNumOutput;
            }
            else
            {
                m_nConvOutChannels = m_nNumOutput;
                m_nConvInChannels = m_nChannels;
            }
 
            // Handle the parameters: weights and biases
            // - blobs[0] holds the filter weights.
            // - blobs[1] holds the biases (optional)
 
            List<int> rgWeightShape = new List<int>();
            rgWeightShape.Add(m_nConvOutChannels);
            rgWeightShape.Add(m_nConvInChannels / m_nGroup);
 
            for (int i = 0; i < m_nNumSpatialAxes; i++)
            {
                rgWeightShape.Add(val_at(rgKernelShape, i));
            }
 
            m_bBiasTerm = p.bias_term;
 
            List<int> rgBiasShape = new List<int>() { m_nNumOutput };
 
            // Setup the convert to half flags used by the Layer just before calling forward and backward.
            if (p.useCudnn(m_nNumSpatialAxes))
                m_bUseHalfSize = m_param.use_halfsize;
 
            if (m_colBlobs.Count > 0)
            {
                m_log.CHECK_EQ(1 + ((m_bBiasTerm) ? 1 : 0), m_colBlobs.Count, "Incorrect number of weight blobs.");
 
                if (!Utility.Compare<int>(rgWeightShape, m_colBlobs[0].shape()))
                {
                    Blob<T> b = new Blob<T>(m_cuda, m_log, rgWeightShape);
                    m_log.FAIL("Incorrect weight shape: expected shape " + b.shape_string + "; instead, shape was " + m_colBlobs[0].shape_string);                   
                }
 
                if (m_bBiasTerm && !Utility.Compare<int>(rgBiasShape, m_colBlobs[1].shape()))
                {
                    Blob<T> b = new Blob<T>(m_cuda, m_log, rgBiasShape);
                    m_log.FAIL("Incorrect bias shape: expected shape " + b.shape_string + "; instead, shape was " + m_colBlobs[1].shape_string);
                }
 
                m_log.WriteLine("Skipping parameter initialization.");
            }
            else
            {
                m_colBlobs.Clear();
 
                // Initialize and fill the weights:
                // output channels x input channels per-group x kernel height x kernel width.
                Blob<T> blobWts = new Blob<T>(m_cuda, m_log, true, m_bUseHalfSize);
                blobWts.Name = m_param.name + " weights";
                blobWts.type = BLOB_TYPE.WEIGHT;
 
                if (m_bUseHalfSize || !shareParameter(blobWts, rgWeightShape))
                {
                    blobWts.Reshape(rgWeightShape, m_bUseHalfSize);
                    Filler<T> wtFiller = Filler<T>.Create(m_cuda, m_log, p.weight_filler);
 
                    Blob<T> blobWts1 = blobWts;
 
                    if (m_bUseHalfSize)
                    {
                        blobWts1 = new Blob<T>(m_cuda, m_log, false, false);
                        blobWts1.ReshapeLike(blobWts);
                    }
 
                    wtFiller.Fill(blobWts1);
 
                    if (m_bUseHalfSize)
                    {
                        blobWts.CopyFrom(blobWts1);
                        blobWts1.Dispose();
                    }
                }
 
                m_colBlobs.Add(blobWts);
 
                // If necessary, initialize and fill the biases:
                if (m_bBiasTerm)
                {
                    Blob<T> blobBias = new Blob<T>(m_cuda, m_log, true, m_bUseHalfSize);
                    blobBias.Name = m_param.name + " bias";
                    blobBias.type = BLOB_TYPE.WEIGHT;
 
                    if (m_bUseHalfSize || !shareParameter(blobBias, rgBiasShape))
                    {
                        blobBias.Reshape(rgBiasShape, m_bUseHalfSize);
                        Filler<T> biasFiller = Filler<T>.Create(m_cuda, m_log, p.bias_filler);
 
                        Blob<T> blobBias1 = blobBias;
 
                        if (m_bUseHalfSize)
                        {
                            blobBias1 = new Blob<T>(m_cuda, m_log, false, false);
                            blobBias1.ReshapeLike(blobBias);
                        }
 
                        biasFiller.Fill(blobBias1);
 
                        if (m_bUseHalfSize)
                        {
                            blobBias.CopyFrom(blobBias1);
                            blobBias1.Dispose();
                        }
                    }
 
                    m_colBlobs.Add(blobBias);
                }
            }
 
            m_nKernelDim = m_colBlobs[0].count(1);
            m_nWeightOffset = m_nConvOutChannels * m_nKernelDim / m_nGroup;
 
            // Propagate gradients to the parameters (as directed by backward pass).
            m_rgbParamPropagateDown = new DictionaryMap<bool>(m_colBlobs.Count, true);
        }
 
        public override void Reshape(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            if (!reshapeNeeded(colBottom, colTop))
                return;
 
            int nFirstSpatialAxis = m_nChannelAxis + 1;
            m_log.CHECK_EQ(colBottom[0].num_axes, nFirstSpatialAxis + m_nNumSpatialAxes, "bottom num_axes may not change.");
 
            m_nNum = colBottom[0].count(0, m_nChannelAxis);
            m_log.CHECK_EQ(colBottom[0].shape(m_nChannelAxis), m_nChannels, "Input size incompatible with convolution kernel.");
 
            // TODO: generalize to handle inputs of different shapes.
            for (int i = 1; i < colBottom.Count; i++)
            {
                m_log.CHECK(Utility.Compare<int>(colBottom[0].shape(), colBottom[i].shape()), "Shape mismatch - bottom[0]: '" + colBottom[0].shape_string + "' vs. bottom[" + i.ToString() + "]: '" + colBottom[i].shape_string + "'");
            }
 
            // Shape the tops.
            m_rgBottomShape = Utility.Clone<int>(colBottom[0].shape());
            compute_output_shape();
 
            List<int> rgTopShape = new List<int>();
 
            for (int i = 0; i < m_nChannelAxis; i++)
            {
                rgTopShape.Add(colBottom[0].shape(i));
            }
 
            rgTopShape.Add(m_nNumOutput);
 
            for (int i = 0; i < m_nNumSpatialAxes; i++)
            {
                rgTopShape.Add(m_rgOutputShape[i]);
            }
 
            for (int i = 0; i < colTop.Count; i++)
            {
                colTop[i].Reshape(rgTopShape, m_bUseHalfSize);
            }
 
            if (reverse_dimensions())
                m_nConvOutSpatialDim = colBottom[0].count(nFirstSpatialAxis);
            else
                m_nConvOutSpatialDim = colTop[0].count(nFirstSpatialAxis);
 
            m_nColOffset = m_nKernelDim * m_nConvOutSpatialDim;
            m_nOutputOffset = m_nConvOutChannels * m_nConvOutSpatialDim / m_nGroup;
 
            if (!m_param.convolution_param.useCudnn(m_nNumSpatialAxes) || reverse_dimensions())
            {
                // Setup input dimensions (blobConvInputShape)
                List<int> rgBottomDimBlobShape = new List<int>() { m_nNumSpatialAxes + 1 };
                m_blobConvInputShape.Reshape(rgBottomDimBlobShape);
 
                T[] rgConvInputShapeData = m_blobConvInputShape.mutable_cpu_data;
                for (int i = 0; i < m_nNumSpatialAxes + 1; i++)
                {
                    if (reverse_dimensions())
                        rgConvInputShapeData[i] = (T)Convert.ChangeType(colTop[0].shape(m_nChannelAxis + i), typeof(T));
                    else
                        rgConvInputShapeData[i] = (T)Convert.ChangeType(colBottom[0].shape(m_nChannelAxis + i), typeof(T));
                }
                m_blobConvInputShape.mutable_cpu_data = rgConvInputShapeData;
 
                // The im2col result buffer will only hold one image at a time to avoid
                // overly large memory usage.  In the special case of 1x1 convolution
                // it goes lazily unused to save memory.
                m_rgColBufferShape = new List<int>();
                m_rgColBufferShape.Add(m_nKernelDim * m_nGroup);
 
                for (int i = 0; i < m_nNumSpatialAxes; i++)
                {
                    if (reverse_dimensions())
                        m_rgColBufferShape.Add(input_shape(i + 1));
                    else
                        m_rgColBufferShape.Add(m_rgOutputShape[i]);
                }
 
                shareLayerBlob(m_blobColBuffer, m_rgColBufferShape);
                m_blobColBuffer.Reshape(m_rgColBufferShape);
            }
 
            m_nBottomDim = colBottom[0].count(m_nChannelAxis);
            m_nTopDim = colTop[0].count(m_nChannelAxis);
            m_nNumKernelsIm2col = m_nConvInChannels * m_nConvOutSpatialDim;
            m_nNumKernelsCol2im = (reverse_dimensions()) ? m_nTopDim : m_nBottomDim;
 
            // Setup up the all ones 'bias_multiplier' for adding biases by BLAS
            m_nOutSpatialDim = colTop[0].count(nFirstSpatialAxis);
 
            if (m_bBiasTerm)
            {
                if (!m_param.convolution_param.useCudnn(m_nNumSpatialAxes) || reverse_dimensions())
                {
                    List<int> rgBiasMultShape = new List<int>() { m_nOutSpatialDim };
                    shareLayerBlob(m_blobBiasMultiplier, rgBiasMultShape);
                    m_blobBiasMultiplier.Reshape(rgBiasMultShape);
                    m_blobBiasMultiplier.SetData(1.0);
                }
            }
        }
 
        public override int MinBottomBlobs
        {
            get { return 1; }
        }
 
        public override int MinTopBlobs
        {
            get { return 1; }
        }
 
        public override bool EqualNumBottomTopBlobs
        {
            get { return true; }
        }
 
        protected void forward_gemm(long hInput, int nInputOffset, long hWeights, long hOutput, int nOutputOffset, bool bSkipIm2Col = false)
        {
            long hColBuff = hInput;
            int nColBuffOffset = nInputOffset;
 
            if (!m_bIs1x1)
            {
                if (!bSkipIm2Col)
                    conv_im2col(hInput, nInputOffset, m_blobColBuffer.mutable_gpu_data, 0);
 
                hColBuff = m_blobColBuffer.gpu_data;
                nColBuffOffset = 0;
            }
 
            //m_cuda.gemm(false, false, m_nConvOutChannels / m_nGroup, m_nConvOutSpatialDim, m_nKernelDim, m_tOne, hWeights, hColBuff, m_tZero, hOutput, m_nWeightOffset * g, nColBuffOffset + m_nColOffset * g, nOutputOffset + m_nOutputOffset * g); 
            m_cuda.gemm(false, false, m_nConvOutChannels / m_nGroup, m_nConvOutSpatialDim, m_nKernelDim, m_tOne, hWeights, hColBuff, m_tZero, hOutput, 0, nColBuffOffset, nOutputOffset, m_nGroup, m_nWeightOffset, m_nColOffset, m_nOutputOffset); 
        }
 
        protected void forward_bias(long hOutput, int nOutputOffset, long hBias)
        {
            m_cuda.gemm(false, false, m_nNumOutput, m_nOutSpatialDim, 1, m_tOne, hBias, m_blobBiasMultiplier.gpu_data, m_tOne, hOutput, 0, 0, nOutputOffset);
        }
 
        protected void backward_gemm(long hOutput, int nOutputOffset, long hWeights, long hInput, int nInputOffset)
        {
            long hColBuff = m_blobColBuffer.mutable_gpu_data;
            int nColBuffOffset = 0;
 
            if (m_bIs1x1)
            {
                hColBuff = hInput;
                nColBuffOffset = nInputOffset;
            }
 
            //for (int g = 0; g < m_nGroup; g++)
            //{
            //    m_cuda.gemm(true, false, m_nKernelDim, m_nConvOutSpatialDim, m_nConvOutChannels / m_nGroup, m_tOne, hWeights, hOutput, m_tZero, hColBuff, m_nWeightOffset * g, nOutputOffset + m_nOutputOffset * g, nColBuffOffset + m_nColOffset * g);
            //}
 
            m_cuda.gemm(true, false, m_nKernelDim, m_nConvOutSpatialDim, m_nConvOutChannels / m_nGroup, m_tOne, hWeights, hOutput, m_tZero, hColBuff, 0, nOutputOffset, nColBuffOffset, m_nGroup, m_nWeightOffset, m_nOutputOffset, m_nColOffset);
 
            if (!m_bIs1x1)
                conv_col2im(hColBuff, nColBuffOffset, hInput, nInputOffset);
        }
 
        protected void weight_gemm(long hInput, int nInputOffset, long hOutput, int nOutputOffset, long hWeights)
        {
            long hColBuff = hInput;
            int nColBuffOffset = nInputOffset;
 
            if (!m_bIs1x1)
            {
                conv_im2col(hInput, nInputOffset, m_blobColBuffer.mutable_gpu_data, 0);
                hColBuff = m_blobColBuffer.gpu_data;
                nColBuffOffset = 0;
            }
 
            //for (int g = 0; g < m_nGroup; g++)
            //{
            //    m_cuda.gemm(false, true, m_nConvOutChannels / m_nGroup, m_nKernelDim, m_nConvOutSpatialDim, m_tOne, hOutput, hColBuff, m_tOne, hWeights, nOutputOffset + m_nOutputOffset * g, nColBuffOffset + m_nColOffset * g, m_nWeightOffset * g);
            //}
 
            m_cuda.gemm(false, true, m_nConvOutChannels / m_nGroup, m_nKernelDim, m_nConvOutSpatialDim, m_tOne, hOutput, hColBuff, m_tOne, hWeights, nOutputOffset, nColBuffOffset, 0, m_nGroup, m_nOutputOffset, m_nColOffset, m_nWeightOffset);
        }
 
        protected void backward_bias(long hBias, long hInput, int nInputOffset)
        {
            m_cuda.gemv(false, m_nNumOutput, m_nOutSpatialDim, m_tOne, hInput, m_blobBiasMultiplier.gpu_data, m_tOne, hBias, nInputOffset, 0, 0);
        }
 
        protected int input_shape(int i)
        {
            return m_rgBottomShape[m_nChannelAxis + i];
        }
 
        protected abstract bool reverse_dimensions();
 
        protected abstract void compute_output_shape();
 
        private void conv_im2col(long hData, int nDataOffset, long hColBuff, int nColBuffOffset)
        {
            if (!m_bForceNDim2col && m_nNumSpatialAxes == 2)
            {
                T[] rgConvInputShape = m_blobConvInputShape.update_cpu_data();
                T[] rgKernelShape = m_blobKernelShape.update_cpu_data();
                T[] rgPad = m_blobPad.update_cpu_data();
                T[] rgStride = m_blobStride.update_cpu_data();
                T[] rgDilation = m_blobDilation.update_cpu_data();
 
                m_cuda.im2col(hData, 
                              nDataOffset, 
                              m_nConvInChannels,
                              val_at(rgConvInputShape, 1),
                              val_at(rgConvInputShape, 2),
                              val_at(rgKernelShape, 0),
                              val_at(rgKernelShape, 1),
                              val_at(rgPad, 0),
                              val_at(rgPad, 1),
                              val_at(rgStride, 0),
                              val_at(rgStride, 1),
                              val_at(rgDilation, 0),
                              val_at(rgDilation, 1),
                              hColBuff, 
                              nColBuffOffset);
            }
            else
            {
                m_cuda.im2col_nd(hData, 
                              nDataOffset, 
                              m_nNumSpatialAxes, 
                              m_nNumKernelsIm2col,
                              0,
                              m_blobConvInputShape.gpu_data, 
                              m_blobColBuffer.gpu_shape, 
                              m_blobKernelShape.gpu_data, 
                              m_blobPad.gpu_data, 
                              m_blobStride.gpu_data, 
                              m_blobDilation.gpu_data,
                              hColBuff, 
                              nColBuffOffset);
            }
        }
 
        private void conv_col2im(long hColBuff, int nColBuffOffset, long hData, int nDataOffset)
        {
            if (!m_bForceNDim2col && m_nNumSpatialAxes == 2)
            {
                T[] rgConvInputShape = m_blobConvInputShape.update_cpu_data();
                T[] rgKernelShape = m_blobKernelShape.update_cpu_data();
                T[] rgPad = m_blobPad.update_cpu_data();
                T[] rgStride = m_blobStride.update_cpu_data();
                T[] rgDilation = m_blobDilation.update_cpu_data();
 
                m_cuda.col2im(hColBuff,
                              nColBuffOffset, 
                              m_nConvInChannels, 
                              val_at(rgConvInputShape, 1),
                              val_at(rgConvInputShape, 2),
                              val_at(rgKernelShape, 0),
                              val_at(rgKernelShape, 1),
                              val_at(rgPad, 0),
                              val_at(rgPad, 1),
                              val_at(rgStride, 0),
                              val_at(rgStride, 1),
                              val_at(rgDilation, 0),
                              val_at(rgDilation, 1),
                              hData, 
                              nDataOffset);
            }
            else
            {
                m_cuda.col2im_nd(hColBuff, 
                              nColBuffOffset, 
                              m_nNumSpatialAxes, 
                              m_nNumKernelsCol2im,
                              0,
                              m_blobConvInputShape.gpu_data, 
                              m_blobColBuffer.gpu_shape, 
                              m_blobKernelShape.gpu_data, 
                              m_blobPad.gpu_data, 
                              m_blobStride.gpu_data, 
                              m_blobDilation.gpu_data,
                              hData, 
                              nDataOffset);
            }
        }
    }
}