MultiHeadAttentionInterpParameter.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.ComponentModel;
using MyCaffe.basecode;
 
namespace MyCaffe.param.tft
{
    [Serializable]
    [TypeConverter(typeof(ExpandableObjectConverter))]
    public class MultiHeadAttentionInterpParameter : LayerParameterBase
    {
        FillerParameter m_fillerParam_weights = new FillerParameter("xavier");
        FillerParameter m_fillerParam_bias = new FillerParameter("constant", 0.1);
        bool m_bEnableNoise = false;
        double m_dfSigmaInit = 0.017;
        uint m_nEmbedDim;
        uint m_nNumHeads;
        uint m_nNumHistoricalSteps = 0;  
        uint m_nNumFutureSteps = 0;
        bool m_bEnableSelfAttention = true;
 
        public MultiHeadAttentionInterpParameter()
        {
        }
 
        [Description("Specifies to enable self attention (one input, default = true).")]
        public bool enable_self_attention
        {
            get { return m_bEnableSelfAttention; }
            set { m_bEnableSelfAttention = value; }
        }
 
        [Description("Specifies the number of historical steps.")]
        public uint num_historical_steps
        {
            get { return m_nNumHistoricalSteps; }
            set { m_nNumHistoricalSteps = value; }
        }
 
        [Description("Specifies the number of future steps.")]
        public uint num_future_steps
        {
            get { return m_nNumFutureSteps; }
            set { m_nNumFutureSteps = value; }
        }
 
        [Description("Specifies the state size corresponding to both the input and output sizes.")]
        public uint embed_dim
        {
            get { return m_nEmbedDim; }
            set { m_nEmbedDim = value; }
        }
 
        [Description("Specifies number of attention heads used in the multi-attention.")]
        public uint num_heads
        {
            get { return m_nNumHeads; }
            set { m_nNumHeads = value; }
        }
 
        [Description("Enable/disable noise in the inner-product layer (default = false).")]
        public bool enable_noise
        {
            get { return m_bEnableNoise; }
            set { m_bEnableNoise = value; }
        }
 
        [Description("Specifies the initialization value for the sigma weight and sigma bias used when 'enable_noise' = true.")]
        public double sigma_init
        {
            get { return m_dfSigmaInit; }
            set { m_dfSigmaInit = value; }
        }
 
        [Category("Fillers")]
        [Description("The filler for the weights.")]
        public FillerParameter weight_filler
        {
            get { return m_fillerParam_weights; }
            set { m_fillerParam_weights = value; }
        }
 
        [Category("Fillers")]
        [Description("The filler for the bias.")]
        public FillerParameter bias_filler
        {
            get { return m_fillerParam_bias; }
            set { m_fillerParam_bias = value; }
        }
 
        public override object Load(System.IO.BinaryReader br, bool bNewInstance = true)
        {
            RawProto proto = RawProto.Parse(br.ReadString());
            MultiHeadAttentionInterpParameter p = FromProto(proto);
 
            if (!bNewInstance)
                Copy(p);
 
            return p;
        }
 
        public override void Copy(LayerParameterBase src)
        {
            MultiHeadAttentionInterpParameter p = (MultiHeadAttentionInterpParameter)src;
 
            m_bEnableSelfAttention = p.enable_self_attention;
            m_nNumHistoricalSteps = p.num_historical_steps;
            m_nNumFutureSteps = p.num_future_steps;
 
            m_nEmbedDim = p.embed_dim;
            m_nNumHeads = p.num_heads;
 
            if (p.m_fillerParam_bias != null)
                m_fillerParam_bias = p.m_fillerParam_bias.Clone();
 
            if (p.m_fillerParam_weights != null)
                m_fillerParam_weights = p.m_fillerParam_weights.Clone();
 
            m_bEnableNoise = p.m_bEnableNoise;
            m_dfSigmaInit = p.m_dfSigmaInit;
        }
 
        public override LayerParameterBase Clone()
        {
            MultiHeadAttentionInterpParameter p = new MultiHeadAttentionInterpParameter();
            p.Copy(this);
            return p;
        }
 
        public override RawProto ToProto(string strName)
        {
            RawProtoCollection rgChildren = new RawProtoCollection();
 
            rgChildren.Add("enable_self_attention", enable_self_attention.ToString());
            rgChildren.Add("num_historical_steps", num_historical_steps.ToString());
            rgChildren.Add("num_future_steps", num_future_steps.ToString());
 
            rgChildren.Add("embed_dim", embed_dim.ToString());
            rgChildren.Add("num_heads", num_heads.ToString());
 
            if (weight_filler != null)
                rgChildren.Add(weight_filler.ToProto("weight_filler"));
 
            if (bias_filler != null)
                rgChildren.Add(bias_filler.ToProto("bias_filler"));
 
            if (m_bEnableNoise)
            {
                rgChildren.Add("enable_noise", m_bEnableNoise.ToString());
                rgChildren.Add("sigma_init", m_dfSigmaInit.ToString());
            }
 
            return new RawProto(strName, "", rgChildren);
        }
 
        public static MultiHeadAttentionInterpParameter FromProto(RawProto rp)
        {
            string strVal;
            MultiHeadAttentionInterpParameter p = new MultiHeadAttentionInterpParameter();
 
            if ((strVal = rp.FindValue("enable_self_attention")) != null)
                p.enable_self_attention = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("embed_dim")) != null)
                p.embed_dim = uint.Parse(strVal);
 
            if ((strVal = rp.FindValue("num_heads")) != null)
                p.num_heads = uint.Parse(strVal);
 
            RawProto rpWeightFiller = rp.FindChild("weight_filler");
            if (rpWeightFiller != null)
                p.weight_filler = FillerParameter.FromProto(rpWeightFiller);
 
            RawProto rpBiasFiller = rp.FindChild("bias_filler");
            if (rpBiasFiller != null)
                p.bias_filler = FillerParameter.FromProto(rpBiasFiller);
 
            if ((strVal = rp.FindValue("enable_noise")) != null)
                p.enable_noise = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("sigma_init")) != null)
                p.sigma_init = ParseDouble(strVal);
 
            if ((strVal = rp.FindValue("num_historical_steps")) != null)
                p.num_historical_steps = uint.Parse(strVal);
 
            if ((strVal = rp.FindValue("num_future_steps")) != null)
                p.num_future_steps = uint.Parse(strVal);
 
            return p;
        }
    }
}