TransformerBlockParameter.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.ComponentModel;
using MyCaffe.basecode;
 
namespace MyCaffe.param.gpt
{
    [Serializable]
    [TypeConverter(typeof(ExpandableObjectConverter))]
    public class TransformerBlockParameter : LayerParameterBase
    {
        uint m_nHeads = 6;
        uint m_nEmbed = 192;
        double m_dfAttnDropout = 0.1;
        double m_dfResidDropout = 0.1;
        uint m_nBlockSize = 128;
        uint m_nLayers = 6;
        ACTIVATION m_activation = ACTIVATION.RELU;
        BLOCK_TYPE m_type = BLOCK_TYPE.CAUSAL_SELF_ATTENTION;
        bool m_bEnableLayerNormCudaImplementation = false;
 
        public enum BLOCK_TYPE
        {
            CAUSAL_SELF_ATTENTION = 0,
            ENCODER,
            DECODER
        }
 
        public enum ACTIVATION
        {
            RELU = 0,
            GELU = 1,
            GELU_BERT = 2
        }
 
        public TransformerBlockParameter()
        {
            
        }
 
        public bool enable_layernorm_cuda_impl
        {
            get { return m_bEnableLayerNormCudaImplementation; }
            set { m_bEnableLayerNormCudaImplementation = value; }
        }
 
        public ACTIVATION activation
        {
            get { return m_activation; }
            set { m_activation = value; }
        }
 
        public BLOCK_TYPE block_type
        {
            get { return m_type; }
            set { m_type = value; }
        }
 
        [Description("Specifies number of layers (transformer blocks) used.")]
        public uint layers
        {
            get { return m_nLayers; }
            set { m_nLayers = value; }
        }
 
        [Description("Specifies number of heads used.")]
        public uint heads
        {
            get { return m_nHeads; }
            set { m_nHeads = value; }
        }
 
        public uint embed
        {
            get { return m_nEmbed; }
            set { m_nEmbed = value; }
        }
 
        public uint block_size
        {
            get { return m_nBlockSize; }
            set { m_nBlockSize = value; }
        }
        
        public double attn_dropout
        {
            get { return m_dfAttnDropout; }
            set { m_dfAttnDropout = value; }
        }
 
        public double resid_dropout
        {
            get { return m_dfResidDropout; }
            set { m_dfResidDropout = value; }
        }
 
        public override object Load(System.IO.BinaryReader br, bool bNewInstance = true)
        {
            RawProto proto = RawProto.Parse(br.ReadString());
            TransformerBlockParameter p = FromProto(proto);
 
            if (!bNewInstance)
                Copy(p);
 
            return p;
        }
 
        public override void Copy(LayerParameterBase src)
        {
            TransformerBlockParameter p = (TransformerBlockParameter)src;
 
            m_nLayers = p.layers;
            m_nHeads = p.heads;
            m_nEmbed = p.embed;
            m_nBlockSize = p.block_size;
            m_dfAttnDropout = p.attn_dropout;
            m_dfResidDropout = p.resid_dropout;
            m_activation = p.activation;
            m_type = p.block_type;
            m_bEnableLayerNormCudaImplementation = p.enable_layernorm_cuda_impl;
        }
 
        public override LayerParameterBase Clone()
        {
            TransformerBlockParameter p = new TransformerBlockParameter();
            p.Copy(this);
            return p;
        }
 
        public override RawProto ToProto(string strName)
        {
            RawProtoCollection rgChildren = new RawProtoCollection();
 
            rgChildren.Add("layers", layers.ToString());
            rgChildren.Add("heads", heads.ToString());
            rgChildren.Add("embed", embed.ToString());
            rgChildren.Add("block_size", block_size.ToString());
            rgChildren.Add("attn_dropout", attn_dropout.ToString());
            rgChildren.Add("resid_dropout", resid_dropout.ToString());
            rgChildren.Add("activation", activation.ToString());
            rgChildren.Add("block_type", block_type.ToString());
            rgChildren.Add("enable_ln_cuda_impl", enable_layernorm_cuda_impl.ToString());
 
            return new RawProto(strName, "", rgChildren);
        }
 
        public static TransformerBlockParameter FromProto(RawProto rp)
        {
            string strVal;
            TransformerBlockParameter p = new TransformerBlockParameter();
 
            if ((strVal = rp.FindValue("layers")) != null)
                p.layers = uint.Parse(strVal);
 
            if ((strVal = rp.FindValue("heads")) != null)
                p.heads = uint.Parse(strVal);
            
            if ((strVal = rp.FindValue("embed")) != null)
                p.embed = uint.Parse(strVal);
 
            if ((strVal = rp.FindValue("block_size")) != null)
                p.block_size = uint.Parse(strVal);
 
            if ((strVal = rp.FindValue("attn_dropout")) != null)
                p.attn_dropout = double.Parse(strVal);
 
            if ((strVal = rp.FindValue("resid_dropout")) != null)
                p.resid_dropout = double.Parse(strVal);
 
            if ((strVal = rp.FindValue("activation")) != null)
            {
                if (strVal == ACTIVATION.GELU.ToString())
                    p.activation = ACTIVATION.GELU;
                else if (strVal == ACTIVATION.GELU_BERT.ToString())
                    p.activation = ACTIVATION.GELU_BERT;
                else
                    p.activation = ACTIVATION.RELU;
            }
 
            if ((strVal = rp.FindValue("block_type")) != null)
            {
                if (strVal == BLOCK_TYPE.CAUSAL_SELF_ATTENTION.ToString())
                    p.block_type = BLOCK_TYPE.CAUSAL_SELF_ATTENTION;
                else if (strVal == BLOCK_TYPE.ENCODER.ToString())
                    p.block_type = BLOCK_TYPE.ENCODER;
                else if (strVal == BLOCK_TYPE.DECODER.ToString())
                    p.block_type = BLOCK_TYPE.DECODER;
            }
 
            if ((strVal = rp.FindValue("enable_ln_cuda_impl")) != null)
                p.enable_layernorm_cuda_impl = bool.Parse(strVal);
 
            return p;
        }
    }
}