GeluLayer.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using MyCaffe.basecode;
using MyCaffe.common;
using MyCaffe.param;
 
namespace MyCaffe.layers.gpt
{
    public class GeluLayer<T> : NeuronLayer<T>
    {
        bool m_bEnableBertVersion;
 
        public GeluLayer(CudaDnn<T> cuda, Log log, LayerParameter p)
            : base(cuda, log, p)
        {
            m_type = LayerParameter.LayerType.GELU;
            m_bEnableBertVersion = p.gelu_param.enable_bert_version;
        }
 
        protected override void forward(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            long hBottomData = colBottom[0].gpu_data;
            long hTopData = colTop[0].mutable_gpu_data;
            int nCount = colBottom[0].count();
 
            m_cuda.gelu_fwd(nCount, hBottomData, hTopData, m_bEnableBertVersion);
        }
 
        protected override void backward(BlobCollection<T> colTop, List<bool> rgbPropagateDown, BlobCollection<T> colBottom)
        {
            long hTopData = colTop[0].gpu_data;
            long hTopDiff = colTop[0].gpu_diff;
            long hBottomDiff = colBottom[0].mutable_gpu_diff;
            long hBottomData = colBottom[0].gpu_data;
            int nCount = colBottom[0].count();
 
            m_cuda.gelu_bwd(nCount, hTopDiff, hTopData, hBottomDiff, hBottomData, m_bEnableBertVersion);
        }
    }
}