TokenizedDataParameter.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.ComponentModel;
using MyCaffe.basecode;
using MyCaffe.param.python;
 
namespace MyCaffe.param.gpt
{
    [Serializable]
    [TypeConverter(typeof(ExpandableObjectConverter))]
    public class TokenizedDataParameter : LayerParameterBase
    {
        protected PythonParameter m_pythonParam = new PythonParameter();
        
        uint m_nBatchSize;
        uint m_nBlockSize;
        INPUT_TYPE m_inputType;
        string m_strSource;
        int? m_nSeed = null;
        string m_strDbgIdxFile;
        VOCABULARY_TYPE m_vocabType = VOCABULARY_TYPE.CHARACTER;
        SAMPLE_METHOD m_sampleMethod = SAMPLE_METHOD.ARGMAX;
 
        public enum VOCABULARY_TYPE
        {
            CHARACTER,
            WORD,
            SENTENCEPIECE,
            CUSTOM
        }
 
        public enum SAMPLE_METHOD
        {
            ARGMAX,
            PROBABILITY
        }
 
        public enum INPUT_TYPE
        {
            TEXT_FILE,
            CUSTOM              
        }
 
        public TokenizedDataParameter()
        {            
        }
 
        public PythonParameter python_param
        {
            get { return m_pythonParam; }
            set { m_pythonParam = value; }
        }
 
        public int? seed
        {
            get { return m_nSeed; }
            set { m_nSeed = value; }
        }
 
        [Description("Specifies data source input type.")]
        public INPUT_TYPE input_type
        {
            get { return m_inputType; }
            set { m_inputType = value; }
        }
 
        [Description("Specifies the vocabulary type to use.")]
        public VOCABULARY_TYPE vocabulary_type
        {
            get { return m_vocabType; }
            set { m_vocabType = value; }
        }
 
        [Description("Specifies the sampling method used when post processing logits (default = ARGMAX).")]
        public SAMPLE_METHOD sample_method
        {
            get { return m_sampleMethod; }
            set { m_sampleMethod = value; }
        }
 
        [Description("Specifies the data source based on the INPUT_TYPE used.  Each dataset has both a training and testing data source.")]
        public string source
        {
            get { return m_strSource; }
            set { m_strSource = value; }
        }
 
        [Description("Specifies an optional data index file used for debuging only.")]
        public string debug_index_file
        {
            get { return m_strDbgIdxFile; }
            set { m_strDbgIdxFile = value; }
        }
 
        [Description("Specifies batch size.")]
        public uint batch_size
        {
            get { return m_nBatchSize; }
            set { m_nBatchSize = value; }
        }
 
        public uint block_size
        {
            get { return m_nBlockSize; }
            set { m_nBlockSize = value; }
        }
        
        public override object Load(System.IO.BinaryReader br, bool bNewInstance = true)
        {
            RawProto proto = RawProto.Parse(br.ReadString());
            TokenizedDataParameter p = FromProto(proto);
 
            if (!bNewInstance)
                Copy(p);
 
            return p;
        }
 
        public override void Copy(LayerParameterBase src)
        {
            TokenizedDataParameter p = (TokenizedDataParameter)src;
 
            m_pythonParam = p.python_param;
            m_inputType = p.input_type;
            m_strSource = p.source;
            m_nBatchSize = p.batch_size;
            m_nBlockSize = p.block_size;
            m_nSeed = p.seed;
            m_strDbgIdxFile = p.debug_index_file;
            m_vocabType = p.vocabulary_type;       
            m_sampleMethod = p.sample_method;
        }
 
        public override LayerParameterBase Clone()
        {
            TokenizedDataParameter p = new TokenizedDataParameter();
            p.Copy(this);
            return p;
        }
 
        public override RawProto ToProto(string strName)
        {
            RawProtoCollection rgChildren = new RawProtoCollection();
 
            if (m_pythonParam != null)
                rgChildren.Add(m_pythonParam.ToProto("python_param"));
 
            rgChildren.Add("input_type", input_type.ToString());
            rgChildren.Add("vocabulary_type", vocabulary_type.ToString());
            rgChildren.Add("sample_method", sample_method.ToString());
            rgChildren.Add("source", "\"" + source + "\"");
            rgChildren.Add("batch_size", batch_size.ToString());
            rgChildren.Add("block_size", block_size.ToString());
 
            if (!string.IsNullOrEmpty(debug_index_file))
                rgChildren.Add("debug_index_file", debug_index_file);
 
            if (seed != null)
                rgChildren.Add("seed", seed.ToString());
            
            return new RawProto(strName, "", rgChildren);
        }
 
        public static TokenizedDataParameter FromProto(RawProto rp)
        {
            string strVal;
            TokenizedDataParameter p = new TokenizedDataParameter();
 
            RawProto rpPython = rp.FindChild("python_param");
            if (rpPython != null)
                p.python_param = PythonParameter.FromProto(rpPython);
 
            if ((strVal = rp.FindValue("block_size")) != null)
                p.block_size = uint.Parse(strVal);
 
            if ((strVal = rp.FindValue("batch_size")) != null)
                p.batch_size = uint.Parse(strVal);
            
            if ((strVal = rp.FindValue("source")) != null)
                p.source = strVal.Trim('\"');
 
            if ((strVal = rp.FindValue("seed")) != null)
                p.seed = int.Parse(strVal);
 
            if ((strVal = rp.FindValue("debug_index_file")) != null)
                p.debug_index_file = strVal;
 
            if ((strVal = rp.FindValue("input_type")) != null)
            {
                if (strVal == INPUT_TYPE.TEXT_FILE.ToString())
                    p.input_type = INPUT_TYPE.TEXT_FILE;
                else if (strVal == INPUT_TYPE.CUSTOM.ToString())
                    p.input_type = INPUT_TYPE.CUSTOM;
                else
                    throw new Exception("Unknown input type '" + strVal + "'");
            }
 
            if ((strVal = rp.FindValue("vocabulary_type")) != null)
            {
                if (strVal == VOCABULARY_TYPE.CHARACTER.ToString())
                    p.vocabulary_type = VOCABULARY_TYPE.CHARACTER;
                else if (strVal == VOCABULARY_TYPE.WORD.ToString())
                    p.vocabulary_type = VOCABULARY_TYPE.WORD;
                else if (strVal == VOCABULARY_TYPE.SENTENCEPIECE.ToString())
                    p.vocabulary_type = VOCABULARY_TYPE.SENTENCEPIECE;
                else if (strVal == VOCABULARY_TYPE.CUSTOM.ToString())
                    p.vocabulary_type = VOCABULARY_TYPE.CUSTOM;
                else
                    throw new Exception("Unknown vocabulary type '" + strVal + "'");
            }
 
            if ((strVal = rp.FindValue("sample_method")) != null)
            {
                if (strVal == SAMPLE_METHOD.ARGMAX.ToString())
                    p.sample_method = SAMPLE_METHOD.ARGMAX;
                else if (strVal == SAMPLE_METHOD.PROBABILITY.ToString())
                    p.sample_method = SAMPLE_METHOD.PROBABILITY;
                else
                    throw new Exception("Unknown sample method '" + strVal + "'");
            }
 
            return p;
        }
    }
}