MyCaffeTrainerRL.cs

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using MyCaffe.basecode;
using MyCaffe.basecode.descriptors;
using MyCaffe.common;
using MyCaffe.gym;
 
namespace MyCaffe.trainers
{
    public partial class MyCaffeTrainerRL : Component, IXMyCaffeCustomTrainerRL, IxTrainerCallback
    {
        protected CryptoRandom m_random = new CryptoRandom();
        protected PropertySet m_properties = null;
        protected int m_nProjectID = 0;
        protected ConnectInfo m_dsCi = null;
        IxTrainerRL m_itrainer = null;
        double m_dfExplorationRate = 0;
        double m_dfOptimalSelectionRate = 0;
        double m_dfGlobalRewards = 0;
        double m_dfGlobalRewardsAve = 0;
        double m_dfGlobalRewardsMax = -double.MaxValue;
        int m_nGlobalEpisodeCount = 0;
        int m_nGlobalEpisodeMax = 0;
        double m_dfLoss = 0;
        int m_nThreads = 1;
        REWARD_TYPE m_rewardType = REWARD_TYPE.MAXIMUM;
        TRAINER_TYPE m_trainerType = TRAINER_TYPE.PG_ST;
        int m_nItertions = -1;
        IXMyCaffeCustomTrainerCallback m_icallback = null;
        int m_nSnapshot = 0;
        bool m_bSnapshot = false;
        object m_syncObj = new object();
 
        enum TRAINER_TYPE
        {
            PG_MT,
            PG_ST,
            PG_SIMPLE
        }
 
        enum REWARD_TYPE
        {
            VALUE,
            AVERAGE,
            MAXIMUM
        }
 
        public MyCaffeTrainerRL()
        {
            InitializeComponent();
        }
 
        public MyCaffeTrainerRL(IContainer container)
        {
            container.Add(this);
 
            InitializeComponent();
        }
 
        #region Overrides
 
        protected virtual string name
        {
            get { return "MyCaffe RL Trainer"; }
        }
 
        protected virtual TRAINING_CATEGORY category
        {
            get { return TRAINING_CATEGORY.REINFORCEMENT; }
        }
 
        protected virtual DatasetDescriptor get_dataset_override(int nProjectID, ConnectInfo ci = null)
        {
            return null;
        }
 
        protected virtual string get_information()
        {
            return "";
        }
 
        protected virtual IxTrainerRL create_trainerD(Component caffe)
        {
            MyCaffeControl<double> mycaffe = caffe as MyCaffeControl<double>;
            m_nProjectID = mycaffe.CurrentProject.OriginalID;
            m_dsCi = mycaffe.DatasetConnectInfo;
 
            int.TryParse(mycaffe.CurrentProject.GetSolverSetting("max_iter"), out m_nItertions);
            int.TryParse(mycaffe.CurrentProject.GetSolverSetting("snapshot"), out m_nSnapshot);
 
            switch (m_trainerType)
            {
                case TRAINER_TYPE.PG_SIMPLE:
                    return new pg.simple.TrainerPG<double>(mycaffe, m_properties, m_random, this);
 
                case TRAINER_TYPE.PG_ST:
                    return new pg.st.TrainerPG<double>(mycaffe, m_properties, m_random, this);
 
                case TRAINER_TYPE.PG_MT:
                    return new pg.mt.TrainerPG<double>(mycaffe, m_properties, m_random, this);
 
                default:
                    throw new Exception("Unknown trainer type '" + m_trainerType.ToString() + "'!");
            }
        }
 
        protected virtual IxTrainerRL create_trainerF(Component caffe)
        {
            MyCaffeControl<float> mycaffe = caffe as MyCaffeControl<float>;
            m_nProjectID = mycaffe.CurrentProject.OriginalID;
            m_dsCi = mycaffe.DatasetConnectInfo;
 
            int.TryParse(mycaffe.CurrentProject.GetSolverSetting("max_iter"), out m_nItertions);
            int.TryParse(mycaffe.CurrentProject.GetSolverSetting("snapshot"), out m_nSnapshot);
 
            switch (m_trainerType)
            {
                case TRAINER_TYPE.PG_SIMPLE:
                    return new pg.simple.TrainerPG<float>(mycaffe, m_properties, m_random, this);
 
                case TRAINER_TYPE.PG_ST:
                    return new pg.st.TrainerPG<float>(mycaffe, m_properties, m_random, this);
 
                case TRAINER_TYPE.PG_MT:
                    return new pg.mt.TrainerPG<float>(mycaffe, m_properties, m_random, this);
 
                default:
                    throw new Exception("Unknown trainer type '" + m_trainerType.ToString() + "'!");
            }
        }
 
        protected virtual void dispose()
        {
        }
 
        protected virtual void initialize(InitializeArgs e)
        {
        }
 
        protected virtual void shutdown()
        {
        }
 
        protected virtual bool getData(GetDataArgs e)
        {
            return false;
        }
 
        protected virtual bool get_update_snapshot(out int nIteration, out double dfAccuracy)
        {
            nIteration = GlobalEpisodeCount;
            dfAccuracy = GlobalRewards;
 
            if (m_bSnapshot)
            {
                m_bSnapshot = false;
                return true;
            }
 
            return false;
        }
 
        protected virtual void openUi()
        {
        }
 
        #endregion
 
        #region IXMyCaffeCustomTrainer Interface
 
        public Stage Stage
        {
            get { return Stage.RL; }
        }
 
        public string Name
        {
            get { return name; }
        }
 
        public TRAINING_CATEGORY TrainingCategory
        {
            get { return category; }
        }
 
        public bool GetUpdateSnapshot(out int nIteration, out double dfAccuracy)
        {
            return get_update_snapshot(out nIteration, out dfAccuracy);
        }
 
        public DatasetDescriptor GetDatasetOverride(int nProjectID, ConnectInfo ci = null)
        {
            return get_dataset_override(nProjectID, ci);
        }
 
        public bool IsTrainingSupported
        {
            get { return true; }
        }
 
        public bool IsTestingSupported
        {
            get { return true; }
        }
 
        public bool IsRunningSupported
        {
            get { return true; }
        }
 
        public void CleanUp()
        {
            cleanup(3000, true);
        }
 
        private void cleanup(int nWait, bool bCallShutdown = false)
        {
            lock (m_syncObj)
            {
                if (m_itrainer != null)
                {
                    m_itrainer.Shutdown(nWait);
                    m_itrainer = null;
                }
 
                if (bCallShutdown)
                    shutdown();
            }
        }
 
        public void Initialize(string strProperties, IXMyCaffeCustomTrainerCallback icallback)
        {
            m_icallback = icallback;
            m_properties = new PropertySet(strProperties);
            m_nThreads = m_properties.GetPropertyAsInt("Threads", 1);
 
            string strRewardType = m_properties.GetProperty("RewardType", false);
            if (strRewardType == null)
                strRewardType = "VAL";
            else
                strRewardType = strRewardType.ToUpper();
 
            if (strRewardType == "VAL" || strRewardType == "VALUE")
                m_rewardType = REWARD_TYPE.VALUE;
            else if (strRewardType == "AVE" || strRewardType == "AVERAGE")
                m_rewardType = REWARD_TYPE.AVERAGE;
 
            string strTrainerType = m_properties.GetProperty("TrainerType");
 
            switch (strTrainerType)
            {
                case "PG.SIMPLE":   // bare bones model (Sigmoid only)
                    m_trainerType = TRAINER_TYPE.PG_SIMPLE;
                    break;
 
                case "PG.ST":       // single thread (Sigmoid and Softmax)
                    m_trainerType = TRAINER_TYPE.PG_ST;
                    break;
 
                case "PG":
                case "PG.MT":       // multi-thread (Sigmoid and Softmax)
                    m_trainerType = TRAINER_TYPE.PG_MT;
                    break;
 
                default:
                    throw new Exception("Unknown trainer type '" + strTrainerType + "'!");
            }
        }
 
        private IxTrainerRL createTrainer(Component mycaffe)
        {
            IxTrainerRL itrainer = null;
 
            if (mycaffe is MyCaffeControl<double>)
                itrainer = create_trainerD(mycaffe);
            else
                itrainer = create_trainerF(mycaffe);
 
            itrainer.Initialize();
 
            return itrainer;
        }
 
        public ResultCollection RunOne(Component mycaffe, int nDelay = 1000)
        {
            if (m_itrainer == null)
                m_itrainer = createTrainer(mycaffe);
 
            ResultCollection res = m_itrainer.RunOne(nDelay);
            cleanup(50);
 
            return res;
        }
 
        public byte[] Run(Component mycaffe, int nN, out string type)
        {
            if (m_itrainer == null)
                m_itrainer = createTrainer(mycaffe);
 
            PropertySet runProp = null;
            IXMyCaffeCustomTrainerCallbackRNN icallback = m_icallback as IXMyCaffeCustomTrainerCallbackRNN;
            if (icallback != null)
                runProp = icallback.GetRunProperties();
 
            byte[] rgResults = m_itrainer.Run(nN, runProp, out type);
            cleanup(0);
 
            return rgResults;
        }
 
        public void Test(Component mycaffe, int nIterationOverride, ITERATOR_TYPE type = ITERATOR_TYPE.ITERATION)
        {
            if (m_itrainer == null)
                m_itrainer = createTrainer(mycaffe);
 
            if (nIterationOverride == -1)
                nIterationOverride = m_nItertions;
 
            m_itrainer.Test(nIterationOverride, type);
            cleanup(500);
        }
 
        public void Train(Component mycaffe, int nIterationOverride, ITERATOR_TYPE type = ITERATOR_TYPE.ITERATION, TRAIN_STEP step = TRAIN_STEP.NONE)
        {
            if (m_itrainer == null)
                m_itrainer = createTrainer(mycaffe);
 
            if (nIterationOverride == -1)
                nIterationOverride = m_nItertions;
 
            m_itrainer.Train(nIterationOverride, type, step);
            cleanup(1000);
        }
 
        #endregion
 
        public void OnInitialize(InitializeArgs e)
        {
            initialize(e);
        }
 
        public void OnShutdown()
        {
            shutdown();
        }
 
        public void OnGetData(GetDataArgs e)
        {
            getData(e);
        }
 
        public void OnUpdateStatus(GetStatusArgs e)
        {
            m_dfGlobalRewards = e.TotalReward;
            m_dfGlobalRewardsMax = Math.Max(m_dfGlobalRewardsMax, e.TotalReward);
            m_dfGlobalRewardsAve = (1.0 / (double)m_nThreads) * e.TotalReward + ((m_nThreads - 1) / (double)m_nThreads) * m_dfGlobalRewardsAve;
            m_dfExplorationRate = e.ExplorationRate;
            m_dfOptimalSelectionRate = e.OptimalSelectionCoefficient;
 
            if (m_nThreads > 1)
                m_nGlobalEpisodeCount++;
            else
                m_nGlobalEpisodeCount = e.Frames;
 
            m_nGlobalEpisodeMax = e.MaxFrames;
            m_dfLoss = e.Loss;
 
            if (m_icallback != null)
            {
                Dictionary<string, double> rgValues = new Dictionary<string, double>();
                rgValues.Add("GlobalIteration", GlobalEpisodeCount);
                rgValues.Add("GlobalLoss", GlobalLoss);
                rgValues.Add("LearningRate", e.LearningRate);
                rgValues.Add("GlobalAccuracy", GlobalRewards);
                rgValues.Add("Threads", m_nThreads);
                m_icallback.Update(TrainingCategory, rgValues);
            }
 
            e.NewFrameCount = m_nGlobalEpisodeCount;
 
            if (e.Index == 0 && m_nSnapshot > 0 && m_nGlobalEpisodeCount > 0 && (m_nGlobalEpisodeCount % m_nSnapshot) == 0)
                m_bSnapshot = true;
        }
 
        public void OnWait(WaitArgs e)
        {
            Thread.Sleep(e.Wait);
        }
 
        public double GetProperty(string strProp)
        {
            switch (strProp)
            {
                case "GlobalLoss":
                    return GlobalLoss;
 
                case "GlobalRewards":
                    return GlobalRewards;
 
                case "GlobalEpisodeCount":
                    return GlobalEpisodeCount;
 
                case "ExplorationRate":
                    return ExplorationRate;
 
                default:
                    throw new Exception("The property '" + strProp + "' is not supported by the MyCaffeTrainerRNN.");
            }
        }
 
        public double GlobalRewards
        {
            get
            {
                switch (m_rewardType)
                {
                    case REWARD_TYPE.VALUE:
                        return m_dfGlobalRewards;
 
                    case REWARD_TYPE.AVERAGE:
                        return m_dfGlobalRewardsAve;
 
                    default:
                        return (m_dfGlobalRewardsMax == -double.MaxValue) ? 0 : m_dfGlobalRewardsMax;
                }
            }
        }
 
        public double GlobalLoss
        {
            get { return m_dfLoss; }
        }
 
        public int GlobalEpisodeCount
        {
            get { return m_nGlobalEpisodeCount; }
        }
 
        public int GlobalEpisodeMax
        {
            get { return m_nGlobalEpisodeMax; }
        }
 
        public double ExplorationRate
        {
            get { return m_dfExplorationRate; }
        }
 
        public double OptimalSelectionRate
        {
            get { return m_dfOptimalSelectionRate; }
        }
 
        public string Information
        {
            get { return get_information(); }
        }
 
        public void OpenUi()
        {
            openUi();
        }
    }
}