LSTMAttentionParameter.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.ComponentModel;
using MyCaffe.basecode;
 
namespace MyCaffe.param
{
    [Serializable]
    [TypeConverter(typeof(ExpandableObjectConverter))]
    public class LSTMAttentionParameter : LayerParameterBase
    {
        uint m_nNumOutput;
        uint m_nNumIpOutput = 0;
        double m_dfClippingThreshold = 0;
        FillerParameter m_fillerWeights = new FillerParameter("xavier");
        FillerParameter m_fillerBias = new FillerParameter("constant", 0.1);
        bool m_bEnableClockworkForgetGateBias = false;
        bool m_bEnableAttention = false;
 
        public LSTMAttentionParameter()
        {
        }
 
        [Description("Specifies the number of outputs for the layer.")]
        public uint num_output
        {
            get { return m_nNumOutput; }
            set { m_nNumOutput = value; }
        }
 
        [Description("Specifies the number of outputs for the layer.  Note, whenb 0, no inner product is performed.")]
        public uint num_output_ip
        {
            get { return m_nNumIpOutput; }
            set { m_nNumIpOutput = value; }
        }
 
        [Description("Specifies the gradient clipping threshold, default = 0.0 (i.e. no clipping).")]
        public double clipping_threshold
        {
            get { return m_dfClippingThreshold; }
            set { m_dfClippingThreshold = value; }
        }
 
        [Description("Specifies the filler parameters for the weight filler.")]
        [TypeConverter(typeof(ExpandableObjectConverter))]
        public FillerParameter weight_filler
        {
            get { return m_fillerWeights; }
            set { m_fillerWeights = value; }
        }
 
        [Description("Specifies the filler parameters for the bias filler.")]
        [TypeConverter(typeof(ExpandableObjectConverter))]
        public FillerParameter bias_filler
        {
            get { return m_fillerBias; }
            set { m_fillerBias = value; }
        }
 
        [Description("When true, the forget gate bias is set to 5.0 as recommended by [1] Koutnik, J., Greff, K., Gomez, F., Schmidhuber, J., 'A Clockwork RNN', 2014")]
        public bool enable_clockwork_forgetgate_bias
        {
            get { return m_bEnableClockworkForgetGateBias; }
            set { m_bEnableClockworkForgetGateBias = value; }
        }
 
        [Description("When enabled, attention is applied to the input state on each cycle through the LSTM.  Attention is used with encoder/decoder models.")]
        public bool enable_attention
        {
            get { return m_bEnableAttention; }
            set { m_bEnableAttention = value; }
        }
 
        public override object Load(System.IO.BinaryReader br, bool bNewInstance = true)
        {
            RawProto proto = RawProto.Parse(br.ReadString());
            LSTMAttentionParameter p = FromProto(proto);
 
            if (!bNewInstance)
                Copy(p);
 
            return p;
        }
 
        public override void Copy(LayerParameterBase src)
        {
            LSTMAttentionParameter p = (LSTMAttentionParameter)src;
 
            m_nNumOutput = p.m_nNumOutput;
            m_nNumIpOutput = p.m_nNumIpOutput;
            m_dfClippingThreshold = p.m_dfClippingThreshold;
            m_fillerWeights = p.m_fillerWeights.Clone();
            m_fillerBias = p.m_fillerBias.Clone();
            m_bEnableClockworkForgetGateBias = p.m_bEnableClockworkForgetGateBias;
            m_bEnableAttention = p.m_bEnableAttention;
        }
 
        public override LayerParameterBase Clone()
        {
            LSTMAttentionParameter p = new LSTMAttentionParameter();
            p.Copy(this);
            return p;
        }
 
        public override RawProto ToProto(string strName)
        {
            RawProtoCollection rgChildren = new RawProtoCollection();
 
            rgChildren.Add("num_output", m_nNumOutput.ToString());
            rgChildren.Add("num_output_ip", m_nNumIpOutput.ToString());
 
            if (m_dfClippingThreshold != 0)
                rgChildren.Add("clipping_threshold", m_dfClippingThreshold.ToString());
 
            rgChildren.Add(m_fillerWeights.ToProto("weight_filler"));
            rgChildren.Add(m_fillerBias.ToProto("bias_filler"));
 
            if (m_bEnableClockworkForgetGateBias != false)
                rgChildren.Add("enable_clockwork_forgetgate_bias", m_bEnableClockworkForgetGateBias.ToString());
 
            rgChildren.Add("enable_attention", m_bEnableAttention.ToString());
 
            return new RawProto(strName, "", rgChildren);
        }
 
        public static LSTMAttentionParameter FromProto(RawProto rp)
        {
            string strVal;
            LSTMAttentionParameter p = new LSTMAttentionParameter();
 
            if ((strVal = rp.FindValue("num_output")) != null)
                p.num_output = uint.Parse(strVal);
 
            if ((strVal = rp.FindValue("num_output_ip")) != null)
                p.num_output_ip = uint.Parse(strVal);
 
            if ((strVal = rp.FindValue("clipping_threshold")) != null)
                p.clipping_threshold = ParseDouble(strVal);
 
            RawProto rpWeightFiller = rp.FindChild("weight_filler");
            if (rpWeightFiller != null)
                p.weight_filler = FillerParameter.FromProto(rpWeightFiller);
 
            RawProto rpBiasFiller = rp.FindChild("bias_filler");
            if (rpBiasFiller != null)
                p.bias_filler = FillerParameter.FromProto(rpBiasFiller);
 
            if ((strVal = rp.FindValue("enable_clockwork_forgetgate_bias")) != null)
                p.enable_clockwork_forgetgate_bias = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("enable_attention")) != null)
                p.enable_attention = bool.Parse(strVal);
 
            return p;
        }
    }
}