SolverParameter.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.ComponentModel;
using MyCaffe.basecode;
using MyCaffe.common;
using MyCaffe.param.ui;
 
namespace MyCaffe.param
{
    [Serializable]
    [TypeConverter(typeof(ExpandableObjectConverter))]
    public class SolverParameter : BaseParameter
    {
        NetParameter m_paramNet = null;
        NetParameter m_paramTrainNet = null;
        List<NetParameter> m_rgTestNets = new List<NetParameter>();
        NetState m_stateTrain = null;
        List<NetState> m_rgStateTest = new List<NetState>();
        List<int> m_rgTestIter = new List<int>() { 300 };
        int m_nTestInterval = 1000;
        bool m_bTestComputeLoss = false;
        bool m_bTestInitialization = true;
        double m_dfBaseLR = 0.01;
        int m_nDisplay = 0;
        int m_nAverageLoss = 1;
        int m_nMaxIter = 500000;
        int m_nIterSize = 1;
        string m_strLrPolicy = "step";
        double m_dfGamma = 0.1;
        double m_dfPower;
        double m_dfMomentum = 0.0;
        double m_dfWeightDecay = 0.0005;
        string m_strRegularizationType = "L2";
        int m_nStepSize = 100000;
        List<int> m_rgnStepValue = new List<int>();
        double m_dfClipGradients = -1;
        int m_nSnapshot = 10000;
        string m_strSnapshotPrefix = "";
        bool m_bSnapshotDiff = false;
        SnapshotFormat m_snapshotFormat = SnapshotFormat.BINARYPROTO;
        int m_nDeviceID = 1;
        long m_lRandomSeed = -1;
        SolverType m_solverType = SolverType.SGD;
        int m_lbfgs_corrections = 100;
        double m_dfDelta = 1e-8;
        double m_dfMomentum2 = 0.999;
        double m_dfRmsDecay = 0.95;
        double m_dfAdamWDecay = 0.1;
        bool m_bDebugInfo = false;
        bool m_bSnapshotAfterTrain = false;
        string m_strCustomTrainer = null;
        string m_strCustomTrainerProperties = null;
        bool m_bOutputAverageResults = false;
        bool m_bSnapshotIncludeWeights = true;
        bool m_bSnapshotIncludeState = true;
        int m_nAverageAccuracyWindow = 0;
        bool m_bEnableClipGradientOutput = false;
 
        // SSD Parameters
        EvaluationType m_evalType = EvaluationType.CLASSIFICATION;
        ApVersion m_apVersion = ApVersion.INTEGRAL;
        bool m_bShowPerClassResult = false;
 
        public enum EvaluationType
        {
            CLASSIFICATION,
            DETECTION
        }
 
        public enum SnapshotFormat
        {
            BINARYPROTO = 1
        }
 
        public enum SolverType
        {
            SGD = 0,
            NESTEROV = 1,
            ADAGRAD = 2,
            RMSPROP = 3,
            ADADELTA = 4,
            ADAM = 5,
            LBFGS = 6,
            ADAMW = 7,
#pragma warning disable 1591
            _MAX = 8
#pragma warning restore 1591
        }
 
        public enum LearningRatePolicyType
        {
            FIXED,
            STEP,
            EXP,
            INV,
            MULTISTEP,
            POLY,
            SIGMOID
        }
 
        public enum RegularizationType
        {
            NONE,
            L1,
            L2
        }
 
        public SolverParameter()
            : base()
        {
        }
 
        public SolverParameter Clone()
        {
            SolverParameter p = SolverParameter.FromProto(ToProto("clone"));
 
            return p;
        }
 
        [Description("Specifies to average loss results before they are output - this can be faster when there are a lot of results in a cycle.")]
        public bool output_average_results
        {
            get { return m_bOutputAverageResults; }
            set { m_bOutputAverageResults = value; }
        }
 
        [Description("Specifies the custom trainer (if any) used by an external process to provide customized training.")]
        public string custom_trainer
        {
            get { return m_strCustomTrainer; }
            set { m_strCustomTrainer = value; }
        }
 
        [Description("Specifies the custom trainer properties (if any) used by an external process to provide the properties for a customized training.")]
        [Browsable(true)]
        [EditorAttribute(typeof(DictionaryParamEditor), typeof(System.Drawing.Design.UITypeEditor))]
        public string custom_trainer_properties
        {
            get { return m_strCustomTrainerProperties; }
            set { m_strCustomTrainerProperties = Utility.Replace(value, ' ', "[sp]"); }
        }
 
        [Browsable(false)]
        public NetParameter net_param
        {
            get { return m_paramNet; }
            set { m_paramNet = value; }
        }
 
        [Browsable(false)]
        public NetParameter train_net_param
        {
            get { return m_paramTrainNet; }
            set { m_paramTrainNet = value; }
        }
 
        [Browsable(false)]
        public List<NetParameter> test_net_param
        {
            get { return m_rgTestNets; }
            set { m_rgTestNets = value; }
        }
 
        [Browsable(false)]
        public NetState train_state
        {
            get { return m_stateTrain; }
            set { m_stateTrain = value; }
        }
 
        [Browsable(false)]
        public List<NetState> test_state
        {
            get { return m_rgStateTest; }
            set { m_rgStateTest = value; }
        }
 
        [Category("Iterations")]
        [Description("Specifies the number of iterations for each test.")]
        public List<int> test_iter
        {
            get { return m_rgTestIter; }
            set { m_rgTestIter = value; }
        }
 
        [Category("Iterations")]
        [Description("Specifies the number of iterations between two testing phases.")]
        public int test_interval
        {
            get { return m_nTestInterval; }
            set { m_nTestInterval = value; }
        }
 
        [Description("Specifies whether or not to test the compute loss.")]
        public bool test_compute_loss
        {
            get { return m_bTestComputeLoss; }
            set { m_bTestComputeLoss = value; }
        }
 
        [Category("Iterations")]
        [Description("If true, run an initial test pass before the first iteration, ensuring memory availability and printing the starting value of the loss.")]
        public bool test_initialization
        {
            get { return m_bTestInitialization; }
            set { m_bTestInitialization = value; }
        }
 
        [Description("Specifies the base learning rate (default = 0.01).")]
        public double base_lr
        {
            get { return m_dfBaseLR; }
            set { m_dfBaseLR = value; }
        }
       
        [Category("Iterations")]
        [Description("Specifies the number of iterations between displaying information.  If display == 0, no information will be displayed.")]
        public int display
        {
            get { return m_nDisplay; }
            set { m_nDisplay = value; }
        }
 
        [Description("Specifies the loss averaged over the last 'average_loss' iterations.")]
        public int average_loss
        {
            get { return m_nAverageLoss; }
            set { m_nAverageLoss = value; }
        }
 
        [Category("Iterations")]
        [Description("Specifies the maximum number of iterations.")]
        public int max_iter
        {
            get { return m_nMaxIter; }
            set { m_nMaxIter = value; }
        }
 
        [Category("Iterations")]
        [Description("Specifies to accumulate gradients over 'iter_size' x 'batch_size' instances.")]
        public int iter_size
        {
            get { return m_nIterSize; }
            set { m_nIterSize = value; }
        }
 
        [Category("Learning Policy")]
        [DisplayName("lr_policy")]
        [Description("Specifies the learning rate decay policy.  \n 'fixed' - always return base_lr.  \n 'step' - return base_lr * gamma ^ (floor(iter/step)).  \n 'exp' - return base_lr * gamma ^ iter.  \n 'inv' - return base_lr * (1 + gamma * iter) ^ (-power)." +
            "\n 'multistep' - similar to 'step' but allows non-uniform steps defined by stepvalue.  \n 'poly' - the effective learning rate follows a polynomial decay, to be zero by the max_iter, returns base_lr * (1 - iter/max_iter) ^ (power)." +
            "\n 'sigmoid' - the effective learning rate follows a sigmoid decay, returns base_lr * (1/(1 + exp(-gamma * (iter - stepsize)))).")]
        public LearningRatePolicyType LearningRatePolicy
        {
            get
            {
                switch (m_strLrPolicy)
                {
                    case "fixed":
                        return LearningRatePolicyType.FIXED;
 
                    case "step":
                        return LearningRatePolicyType.STEP;
 
                    case "exp":
                        return LearningRatePolicyType.EXP;
 
                    case "inv":
                        return LearningRatePolicyType.INV;
 
                    case "multistep":
                        return LearningRatePolicyType.MULTISTEP;
 
                    case "sigmoid":
                        return LearningRatePolicyType.SIGMOID;
 
                    case "poly":
                        return LearningRatePolicyType.POLY;
 
                    default:
                        throw new Exception("Unknown learning rate policy '" + m_strLrPolicy + "'");
                }
            }
            set
            {
                switch (value)
                {
                    case LearningRatePolicyType.FIXED:
                        m_strLrPolicy = "fixed";
                        break;
 
                    case LearningRatePolicyType.STEP:
                        m_strLrPolicy = "step";
                        break;
 
                    case LearningRatePolicyType.EXP:
                        m_strLrPolicy = "exp";
                        break;
 
                    case LearningRatePolicyType.INV:
                        m_strLrPolicy = "inv";
                        break;
 
                    case LearningRatePolicyType.MULTISTEP:
                        m_strLrPolicy = "multistep";
                        break;
 
                    case LearningRatePolicyType.SIGMOID:
                        m_strLrPolicy = "sigmoid";
                        break;
 
                    case LearningRatePolicyType.POLY:
                        m_strLrPolicy = "poly";
                        break;
 
                    default:
                        throw new Exception("Unknown learning rate policy '" + value.ToString() + "'.");
                }
            }
        }
 
        [Browsable(false)]
        public string lr_policy
        {
            get { return m_strLrPolicy; }
            set { m_strLrPolicy = value; }
        }
 
        [Category("Learning Policy")]
        [Description("Specifies the 'gamma' parameter to compute the 'step', 'exp', 'inv', and 'sigmoid' learning policy (default = 0.1).")]
        public double gamma
        {
            get { return m_dfGamma; }
            set { m_dfGamma = value; }
        }
 
        [Category("Learning Policy")]
        [Description("Specifies the 'power' parameter to compute the 'inv' and 'poly' learning policy.")]
        public double power
        {
            get { return m_dfPower; }
            set { m_dfPower = value; }
        }
 
        [Category("Solver - Not AdaGrad or RMSProp")]
        [Description("Specifies the momentum value - used by all solvers EXCEPT the 'AdaGrad' and 'RMSProp' solvers.  For these latter solvers, momentum should = 0.")]
        public double momentum
        {
            get { return m_dfMomentum; }
            set { m_dfMomentum = value; }
        }
 
        [Description("Specifies the weight decay (default = 0.0005).")]
        public double weight_decay
        {
            get { return m_dfWeightDecay; }
            set { m_dfWeightDecay = value; }
        }
 
        [DisplayName("regularization_type")]
        [Description("Specifies the regularization type (default = L2).  The regulation types supported are 'L1' and 'L2' controlled by weight decay.")]
        public RegularizationType Regularization
        {
            get
            {
                switch (m_strRegularizationType)
                {
                    case "NONE":
                        return RegularizationType.NONE;
                        
                    case "L1":
                        return RegularizationType.L1;
 
                    case "L2":
                        return RegularizationType.L2;
 
                    default:
                        throw new Exception("Unknown regularization type '" + m_strRegularizationType + "'");
                }
            }
            set
            {
                switch (value)
                {
                    case RegularizationType.NONE:
                        m_strRegularizationType = "NONE";
                        break;
                        
                    case RegularizationType.L1:
                        m_strRegularizationType = "L1";
                        break;
 
                    case RegularizationType.L2:
                        m_strRegularizationType = "L2";
                        break;
 
                    default:
                        throw new Exception("Unknown regularization type '" + value.ToString() + "'");
                }
            }
        }
 
        [Description("Specifies the regularization type (default = 'L2').  The regulation types supported are 'L1' and 'L2' controlled by weight decay.")]
        [Browsable(false)]
        public string regularization_type
        {
            get { return m_strRegularizationType; }
            set { m_strRegularizationType = value; }
        }
 
        [Category("Learning Policy")]
        [Description("Specifies the stepsize for the learning rate policy 'step'.")]
        public int stepsize
        {
            get { return m_nStepSize; }
            set { m_nStepSize = value; }
        }
 
        [Category("Learning Policy")]
        [Description("Specifies the step values for the learning rate policy 'multistep'.")]
        public List<int> stepvalue
        {
            get { return m_rgnStepValue; }
            set { m_rgnStepValue = value; }
        }
 
        [Description("Set 'clip_gradients' to >= 0 to clip parameter gradients to that L2 norm, whenever their actual LT norm is larger.")]
        public double clip_gradients
        {
            get { return m_dfClipGradients; }
            set { m_dfClipGradients = value; }
        }
 
        [Description("Optionally, enable/disable output status when gradients are clipped (default = true).")]
        public bool enable_clip_gradient_status
        {
            get { return m_bEnableClipGradientOutput; }
            set { m_bEnableClipGradientOutput = value; }
        }
 
        [Category("Snapshot")]
        [Description("Sepcifies the snapshot interval.")]
        public int snapshot
        {
            get { return m_nSnapshot; }
            set { m_nSnapshot = value; }
        }
 
        [Description("Specifies the snapshot prefix.")]
        [Browsable(false)]
        public string snapshot_prefix
        {
            get { return m_strSnapshotPrefix; }
            set { m_strSnapshotPrefix = value; }
        }
 
        [Category("Snapshot")]
        [Description("Specifies whether ot snapshot diff in the results or not.  Snapshotting diff may help debugging but the final snapshot data size will be much larger.")]
        public bool snapshot_diff
        {
            get { return m_bSnapshotDiff; }
            set { m_bSnapshotDiff = value; }
        }
 
        [Description("Specifies the snapshot format.")]
        [Browsable(false)]
        public SnapshotFormat snapshot_format
        {
            get { return m_snapshotFormat; }
            set { m_snapshotFormat = value; }
        }
 
        [Category("Snapshot")]
        [Description("Specifies whether or not the snapshot includes the trained weights.  The default = 'true'.")]
        public bool snapshot_include_weights
        {
            get { return m_bSnapshotIncludeWeights; }
            set { m_bSnapshotIncludeWeights = value; }
        }
 
        [Category("Snapshot")]
        [Description("Specifies whether or not the snapshot includes the solver state.  The default = 'false'.  Including the solver state will slow down the time of each snapshot.")]
        public bool snapshot_include_state
        {
            get { return m_bSnapshotIncludeState; }
            set { m_bSnapshotIncludeState = value; }
        }
 
        [Description("Specifies the device ID that will be used when run on the GPU.")]
        [Browsable(false)]
        public int device_id
        {
            get { return m_nDeviceID; }
            set { m_nDeviceID = value; }
        }
 
        [Description("If non-negative, the seed with which the Solver will initialize the caffe random number generator -- useful for reproducible results.  Otherwise (and by default), initialize using a seed derived from the system clock.")]
        public long random_seed
        {
            get { return m_lRandomSeed; }
            set { m_lRandomSeed = value; }
        }
 
        [Category("Solver")]
        [Description("Specifies the solver type. \n" +
                     "  SGD - stochastic gradient descent with momentum updates weights by a linear combination of the negative gradient and the previous weight update. \n" +
                     "  NESTEROV - Nesterov's accelerated gradient, similar to SGD, but error gradient is computed on the weights with added momentum. \n" +
                     "  ADADELTA - Gradient based optimization like SGD, see M. Zeiler 'Adadelta, An adaptive learning rate method', arXiv preprint, 2012. \n" +
                     "  ADAGRAD - Gradient based optimization like SGD that tries to find rarely seen features, see Duchi, E, and Y. Singer, 'Adaptive subgradient methods for online learning and stochastic optimization', The Journal of Machine Learning Research, 2011. \n" +
                     "  ADAM - Gradient based optimization like SGD that includes 'adaptive momentum estimation' and can be thougth of as a generalization of AdaGrad, see D. Kingma, J. Ba, 'Adam: A method for stochastic optimization', Intl' Conference for Learning Representations, 2015. \n" +
                     "  RMSPROP - Gradient based optimization like SGD, see T. Tieleman, and G. Hinton, 'RMSProp: Divide the gradient by a runnign average of its recent magnitude', COURSERA: Neural Networks for Machine Learning. Technical Report, 2012. \n" + 
                     "  LBGFS - Gradient based on minFunc, see Marc Schmidt 'minFunc'")]
        public SolverType type
        {
            get { return m_solverType; }
            set { m_solverType = value; }
        }
 
        [Category("Solver - Ada, Adam and RMSProp")]
        [Description("Specifies the numerical stability for 'RMSProp', 'AdaGrad', 'AdaDelta' and 'Adam' solvers (default = 1e-08).")]
        public double delta
        {
            get { return m_dfDelta; }
            set { m_dfDelta = value; }
        }
 
        [Category("Solver - Adam")]
        [Description("Specifies an additional momentum property used by the 'Adam' and 'AdamW' solvers (default = 0.999).")]
        public double momentum2
        {
            get { return m_dfMomentum2; }
            set { m_dfMomentum2 = value; }
        }
 
        [Category("Solver - RMSProp")]
        [Description("Specifies the 'RMSProp' decay value used by the 'RMSProp' solver (default = 0.95).  MeanSquare(t) = 'rms_decay' * MeanSquare(t-1) + (1 - 'rms_decay') * SquareGradient(t).  The 'rms_decay' is only used by the 'RMSProp' solver.")]
        public double rms_decay
        {
            get { return m_dfRmsDecay; }
            set { m_dfRmsDecay = value; }
        }
 
        [Category("Solver - AdamW")]
        [Description("Specifies the 'AdamW' detached weight decay value used by the 'AdamW' solver (default = 0.1).")]
        public double adamw_decay
        {
            get { return m_dfAdamWDecay; }
            set { m_dfAdamWDecay = value; }
        }
 
        [Category("Debug")]
        [Description("If true, print information about the sate of the net that may help with debugging learning problems.")]
        public bool debug_info
        {
            get { return m_bDebugInfo; }
            set { m_bDebugInfo = value; }
        }
 
        [Category("Solver - L-BGFS")]
        [Description("Specifies the 'L-BGFS' corrections.")]
        public int lbgfs_corrections
        {
            get { return m_lbfgs_corrections; }
            set { m_lbfgs_corrections = value; }
        }
 
        [Category("Snapshot")]
        [Description("If false, don't save a snapshot after training finishes.")]
        public bool snapshot_after_train
        {
            get { return m_bSnapshotAfterTrain; }
            set { m_bSnapshotAfterTrain = value; }
        }
 
        [Category("SSD")]
        [Description("Specifies the evaluation type to use when using Single-Shot Detection (SSD) - (default = NONE, SSD not used).")]
        public EvaluationType eval_type
        {
            get { return m_evalType; }
            set { m_evalType = value; }
        }
 
        [Category("SSD")]
        [Description("Specifies the AP Version to use for average precision when using Single-Shot Detection (SSD) - (default = INTEGRAL).")]
        public ApVersion ap_version
        {
            get { return m_apVersion; }
            set { m_apVersion = value; }
        }
 
        [Category("SSD")]
        [Description("Specifies whether or not to display results per class when using Single-Shot Detection (SSD) - (default = false).")]
        public bool show_per_class_result
        {
            get { return m_bShowPerClassResult; }
            set { m_bShowPerClassResult = value; }
        }
 
        [Description("Specifies the window over which to average the accuracies (default = 0, which ignores the averaging).")]
        public int accuracy_average_window
        {
            get { return m_nAverageAccuracyWindow; }
            set { m_nAverageAccuracyWindow = value; }
        }
 
        public override RawProto ToProto(string strName)
        {
            RawProtoCollection rgChildren = new RawProtoCollection();
 
            if (net_param != null)
                rgChildren.Add(net_param.ToProto("net_param"));
 
            if (train_net_param != null)
                rgChildren.Add(train_net_param.ToProto("train_net_param"));
 
            foreach (NetParameter np in test_net_param)
            {
                rgChildren.Add(np.ToProto("test_net_param"));
            }
 
            if (train_state != null)
                rgChildren.Add(train_state.ToProto("train_state"));
 
            foreach (NetState ns in test_state)
            {
                rgChildren.Add(ns.ToProto("test_state"));
            }
 
            rgChildren.Add<int>("test_iter", test_iter);
            rgChildren.Add("test_interval", test_interval.ToString());
            rgChildren.Add("test_compute_loss", test_compute_loss.ToString());
            rgChildren.Add("test_initialization", test_initialization.ToString());
            rgChildren.Add("base_lr", base_lr.ToString());
            rgChildren.Add("display", display.ToString());
            rgChildren.Add("average_loss", average_loss.ToString());
            rgChildren.Add("max_iter", max_iter.ToString());
 
            if (iter_size != 1)
                rgChildren.Add("iter_size", iter_size.ToString());
 
            rgChildren.Add("lr_policy", lr_policy);
 
            if (lr_policy == "step" || lr_policy == "exp" || lr_policy == "inv" || lr_policy == "sigmoid")
                rgChildren.Add("gamma", gamma.ToString());
 
            if (lr_policy == "inv" || lr_policy == "poly")
                rgChildren.Add("power", power.ToString());
 
            rgChildren.Add("momentum", momentum.ToString());
            rgChildren.Add("weight_decay", weight_decay.ToString());
            rgChildren.Add("regularization_type", regularization_type);
 
            if (lr_policy == "step")
                rgChildren.Add("stepsize", stepsize.ToString());
 
            if (lr_policy == "multistep")
                rgChildren.Add<int>("stepvalue", stepvalue);
 
            if (clip_gradients >= 0)
            {
                rgChildren.Add("clip_gradients", clip_gradients.ToString());
                rgChildren.Add("enable_clip_gradient_status", enable_clip_gradient_status.ToString());
            }
                
            rgChildren.Add("snapshot", snapshot.ToString());
 
            if (snapshot_prefix.Length > 0)
                rgChildren.Add("snapshot_prefix", snapshot_prefix);
 
            if (snapshot_diff != false)
                rgChildren.Add("snapshot_diff", snapshot_diff.ToString());
 
            rgChildren.Add("snapshot_format", snapshot_format.ToString());
            rgChildren.Add("device_id", device_id.ToString());
            
            if (random_seed >= 0)
                rgChildren.Add("ransom_seed", random_seed.ToString());
 
            rgChildren.Add("type", type.ToString());
 
            if (type == SolverType.RMSPROP || type == SolverType.ADAGRAD || type == SolverType.ADADELTA || type == SolverType.ADAM || type == SolverType.ADAMW)
                rgChildren.Add("delta", delta.ToString());
 
            if (type == SolverType.ADAM || type == SolverType.ADAMW)
                rgChildren.Add("momentum2", momentum2.ToString());
 
            if (type == SolverType.RMSPROP)
                rgChildren.Add("rms_decay", rms_decay.ToString());
 
            if (type == SolverType.ADAMW)
                rgChildren.Add("adamw_decay", adamw_decay.ToString());
 
            if (type == SolverType.LBFGS)
                rgChildren.Add("lbgfs_corrections", lbgfs_corrections.ToString());
 
            if (debug_info != false)
                rgChildren.Add("debug_info", debug_info.ToString());
   
            if (snapshot_after_train != false)
                rgChildren.Add("snapshot_after_train", snapshot_after_train.ToString());
 
            if (!string.IsNullOrEmpty(custom_trainer))
                rgChildren.Add("custom_trainer", custom_trainer);
 
            if (!string.IsNullOrEmpty(custom_trainer_properties))
                rgChildren.Add("custom_trainer_properties", custom_trainer_properties);
 
            if (output_average_results != false)
                rgChildren.Add("output_average_results", output_average_results.ToString());
 
            rgChildren.Add("snapshot_include_weights", snapshot_include_weights.ToString());
            rgChildren.Add("snapshot_include_state", snapshot_include_state.ToString());
 
            // SSD Parameters
            rgChildren.Add("eval_type", eval_type.ToString().ToLower());
 
            if (ap_version == ApVersion.ELEVENPOINT)
                rgChildren.Add("ap_version", "11point");
            else
                rgChildren.Add("ap_version", ap_version.ToString().ToLower());
 
            rgChildren.Add("show_per_class_result", show_per_class_result.ToString());
            rgChildren.Add("accuracy_average_window", accuracy_average_window.ToString());
 
            return new RawProto(strName, "", rgChildren);
        }
 
        public static SolverParameter FromProto(RawProto rp)
        {
            string strVal;
            SolverParameter p = new SolverParameter();
 
            RawProto rpNetParam = rp.FindChild("net_param");
            if (rpNetParam != null)
                p.net_param = NetParameter.FromProto(rpNetParam);
 
            RawProto rpTrainNetParam = rp.FindChild("train_net_param");
            if (rpTrainNetParam != null)
                p.train_net_param = NetParameter.FromProto(rpTrainNetParam);
 
            RawProtoCollection rgpTn = rp.FindChildren("test_net_param");
            foreach (RawProto rpTest in rgpTn)
            {
                p.test_net_param.Add(NetParameter.FromProto(rpTest));
            }
 
            RawProto rpTrainState = rp.FindChild("train_state");
            if (rpTrainState != null)
                p.train_state = NetState.FromProto(rpTrainState);
 
            RawProtoCollection rgpNs = rp.FindChildren("test_state");
            foreach (RawProto rpNs in rgpNs)
            {
                p.test_state.Add(NetState.FromProto(rpNs));
            }
 
            p.test_iter = rp.FindArray<int>("test_iter");
 
            if ((strVal = rp.FindValue("test_interval")) != null)
                p.test_interval = int.Parse(strVal);
 
            if ((strVal = rp.FindValue("test_compute_loss")) != null)
                p.test_compute_loss = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("test_initialization")) != null)
                p.test_initialization = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("base_lr")) != null)
                p.base_lr = ParseDouble(strVal);
 
            if ((strVal = rp.FindValue("display")) != null)
                p.display = int.Parse(strVal);
 
            if ((strVal = rp.FindValue("average_loss")) != null)
                p.average_loss = int.Parse(strVal);
 
            if ((strVal = rp.FindValue("max_iter")) != null)
                p.max_iter = int.Parse(strVal);
 
            if ((strVal = rp.FindValue("iter_size")) != null)
                p.iter_size = int.Parse(strVal);
 
            if ((strVal = rp.FindValue("lr_policy")) != null)
                p.lr_policy = strVal;
 
            if ((strVal = rp.FindValue("gamma")) != null)
                p.gamma = ParseDouble(strVal);
 
            if ((strVal = rp.FindValue("power")) != null)
                p.power = ParseDouble(strVal);
 
            if ((strVal = rp.FindValue("momentum")) != null)
                p.momentum = ParseDouble(strVal);
 
            if ((strVal = rp.FindValue("weight_decay")) != null)
                p.weight_decay = ParseDouble(strVal);
 
            if ((strVal = rp.FindValue("regularization_type")) != null)
                p.regularization_type = strVal;
 
            if ((strVal = rp.FindValue("stepsize")) != null)
                p.stepsize = int.Parse(strVal);
 
            p.stepvalue = rp.FindArray<int>("stepvalue");
 
            if ((strVal = rp.FindValue("clip_gradients")) != null)
                p.clip_gradients = ParseDouble(strVal);
 
            if ((strVal = rp.FindValue("enable_clip_gradient_status")) != null)
                p.enable_clip_gradient_status = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("snapshot")) != null)
                p.snapshot = int.Parse(strVal);
 
            if ((strVal = rp.FindValue("snapshot_prefix")) != null)
                p.snapshot_prefix = strVal;
 
            if ((strVal = rp.FindValue("snapshot_diff")) != null)
                p.snapshot_diff = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("snapshot_format")) != null)
            {
                switch (strVal)
                {
                    case "BINARYPROTO":
                        p.snapshot_format = SnapshotFormat.BINARYPROTO;
                        break;
 
                    case "HDF5":
                        p.snapshot_format = SnapshotFormat.BINARYPROTO;                        
                        break;
 
                    default:
                        throw new Exception("Unknown 'snapshot_format' value: " + strVal);
                }
            }
 
            if ((strVal = rp.FindValue("device_id")) != null)
                p.device_id = int.Parse(strVal);
 
            if ((strVal = rp.FindValue("random_seed")) != null)
                p.random_seed = long.Parse(strVal);
 
            if ((strVal = rp.FindValue("type")) != null)
            {
                string strVal1 = strVal.ToLower();
 
                switch (strVal1)
                {
                    case "sgd":
                        p.type = SolverType.SGD;
                        break;
 
                    case "nesterov":
                        p.type = SolverType.NESTEROV;
                        break;
 
                    case "adagrad":
                        p.type = SolverType.ADAGRAD;
                        break;
 
                    case "adadelta":
                        p.type = SolverType.ADADELTA;
                        break;
 
                    case "adam":
                        p.type = SolverType.ADAM;
                        break;
 
                    case "adamw":
                        p.type = SolverType.ADAMW;
                        break;
 
                    case "rmsprop":
                        p.type = SolverType.RMSPROP;
                        break;
 
                    case "lbgfs":
                        p.type = SolverType.LBFGS;
                        break;
 
                    default:
                        throw new Exception("Unknown solver 'type' value: " + strVal);
                }
            }
 
            if ((strVal = rp.FindValue("delta")) != null)
                p.delta = ParseDouble(strVal);
 
            if ((strVal = rp.FindValue("momentum2")) != null)
                p.momentum2 = ParseDouble(strVal);
 
            if ((strVal = rp.FindValue("rms_decay")) != null)
                p.rms_decay = ParseDouble(strVal);
 
            if ((strVal = rp.FindValue("adamw_decay")) != null)
                p.adamw_decay = ParseDouble(strVal);
 
            if ((strVal = rp.FindValue("debug_info")) != null)
                p.debug_info = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("lbgfs_corrections")) != null)
                p.lbgfs_corrections = int.Parse(strVal);
 
            if ((strVal = rp.FindValue("snapshot_after_train")) != null)
                p.snapshot_after_train = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("custom_trainer")) != null)
                p.custom_trainer = strVal;
 
            if ((strVal = rp.FindValue("custom_trainer_properties")) != null)
                p.custom_trainer_properties = strVal;
 
            if ((strVal = rp.FindValue("output_average_results")) != null)
                p.output_average_results = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("snapshot_include_weights")) != null)
                p.snapshot_include_weights = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("snapshot_include_state")) != null)
                p.snapshot_include_state = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("eval_type")) != null)
            {
                strVal = strVal.ToLower();
 
                switch (strVal)
                {
                    case "classification":
                        p.eval_type = EvaluationType.CLASSIFICATION;
                        break;
 
                    case "detection":
                        p.eval_type = EvaluationType.DETECTION;
                        break;
 
                    default:
                        throw new Exception("Unknown eval_type '" + strVal + "'!");
                }
            }
 
            if ((strVal = rp.FindValue("ap_version")) != null)
            {
                strVal = strVal.ToLower();
 
                switch (strVal)
                {
                    case "11point":
                        p.ap_version = ApVersion.ELEVENPOINT;
                        break;
 
                    case "maxintegral":
                        p.ap_version = ApVersion.MAXINTEGRAL;
                        break;
 
                    case "integral":
                        p.ap_version = ApVersion.INTEGRAL;
                        break;
 
                    default:
                        throw new Exception("Unknown ap_type '" + strVal + "'!");
                }
            }
 
            if ((strVal = rp.FindValue("show_per_class_result")) != null)
                p.show_per_class_result = bool.Parse(strVal);
 
            if ((strVal = rp.FindValue("accuracy_average_window")) != null)
                p.accuracy_average_window = int.Parse(strVal);
 
            return p;
        }
 
        public string DebugString()
        {
            return m_solverType.ToString();
        }
    }
}