LayerNormParameter.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.ComponentModel;
using MyCaffe.basecode;
 
namespace MyCaffe.param.gpt
{
    [Serializable]
    [TypeConverter(typeof(ExpandableObjectConverter))]
    public class LayerNormParameter : LayerParameterBase 
    {
        double m_dfEpsilon = 1e-10;
        bool m_bEnableCudaImplementation = false;
        bool m_bEnablePassThrough = false;
 
        public LayerNormParameter()
        {
        }
 
        [Description("Specifies the epsilon value used to avoid invalid values (default = 1e-10).")]
        public double epsilon
        {
            get { return m_dfEpsilon; }
            set { m_dfEpsilon = value; }
        }
 
        [Description("Specifies to pass-through the data on the forward and backward pass (e.g. skip the layer norm, used only for debugging. default = false).")]
        public bool enable_passthrough
        {
            get { return m_bEnablePassThrough; }
            set { m_bEnablePassThrough = value; }
        }
 
        [Description("Specifies to use the low-level full cuda implementation of LayerNorm (default = false).")]
        public bool enable_cuda_impl
        {
            get { return m_bEnableCudaImplementation; }
            set { m_bEnableCudaImplementation = value; }
        }
 
        public override object Load(System.IO.BinaryReader br, bool bNewInstance = true)
        {
            RawProto proto = RawProto.Parse(br.ReadString());
            LayerNormParameter p = FromProto(proto);
 
            if (!bNewInstance)
                Copy(p);
 
            return p;
        }
 
        public override void Copy(LayerParameterBase src)
        {
            LayerNormParameter p = (LayerNormParameter)src;
            m_dfEpsilon = p.epsilon;
            m_bEnableCudaImplementation = p.enable_cuda_impl;
            m_bEnablePassThrough = p.enable_passthrough;
        }
 
        public override LayerParameterBase Clone()
        {
            LayerNormParameter p = new LayerNormParameter();
            p.Copy(this);
            return p;
        }
 
        public override RawProto ToProto(string strName)
        {
            RawProtoCollection rgChildren = new RawProtoCollection();
 
            rgChildren.Add("epsilon", m_dfEpsilon.ToString());
            rgChildren.Add("enable_cuda_impl", m_bEnableCudaImplementation.ToString());
 
            if (m_bEnablePassThrough)
                rgChildren.Add("enable_passthrough", m_bEnablePassThrough.ToString());
 
            return new RawProto(strName, "", rgChildren);
        }
 
        public static LayerNormParameter FromProto(RawProto rp)
        {
            string strVal;
            LayerNormParameter p = new LayerNormParameter();
 
            if ((strVal = rp.FindValue("epsilon")) != null)
                p.m_dfEpsilon = double.Parse(strVal);
 
            if ((strVal = rp.FindValue("enable_cuda_impl")) != null)
                p.m_bEnableCudaImplementation = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("enable_passthrough")) != null)
                p.m_bEnablePassThrough = bool.Parse(strVal);
 
            return p;
        }
    }
}