MemoryCollection.cs

using MyCaffe.basecode;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
 
namespace MyCaffe.trainers.common
{
    public class MemoryCollection
    {
        int m_nCount = 0;
        int[] m_rgIdx = null;
        double[] m_rgfPriorities = null;
        protected MemoryItem[] m_rgItems;
        protected int m_nNextIdx = 0;
 
        public MemoryCollection(int nMax)
        {
            m_rgItems = new MemoryItem[nMax];
        }
 
        public int[] Indexes
        {
            get { return m_rgIdx; }
            set { m_rgIdx = value; }
        }
 
        public double[] Priorities
        {
            get { return m_rgfPriorities; }
            set { m_rgfPriorities = value; }
        }
 
        public int NextIndex
        {
            get { return m_nNextIdx; }
        }
 
        public int Count
        {
            get { return m_nCount; }
        }
 
        public MemoryItem this[int nIdx]
        {
            get { return m_rgItems[nIdx]; }
        }
 
        public virtual void Add(MemoryItem item)
        {
            m_rgItems[m_nNextIdx] = item;
            m_nNextIdx++;
 
            if (m_nNextIdx == m_rgItems.Length)
                m_nNextIdx = 0;
 
            if (m_nCount < m_rgItems.Length)
                m_nCount++;
        }
 
        public MemoryCollection GetRandomSamples(CryptoRandom random, int nCount)
        {
            if (nCount >= Count)
                return this;
 
            MemoryCollection col = new MemoryCollection(nCount);
 
            while (col.Count < nCount)
            {
                int nIdx = random.Next(Count);
                col.Add(m_rgItems[nIdx]);
            }
 
            return col;
        }
 
        public List<StateBase> GetNextState()
        {
            return m_rgItems.Select(p => p.NextState).ToList();
        }
 
        public List<SimpleDatum> GetNextStateData()
        {
            return m_rgItems.Select(p => p.NextData).ToList();
        }
 
        public List<SimpleDatum> GetNextStateClip()
        {
            if (m_rgItems[0].NextState != null && m_rgItems[0].NextState.Clip != null)
                return m_rgItems.Select(p => p.NextState.Clip).ToList();
 
            return null;
        }
 
        public List<SimpleDatum> GetCurrentStateData()
        {
            return m_rgItems.Select(p => p.CurrentData).ToList();
        }
 
        public List<SimpleDatum> GetCurrentStateClip()
        {
            if (m_rgItems[0].CurrentState != null && m_rgItems[0].CurrentState.Clip != null)
                return m_rgItems.Select(p => p.CurrentState.Clip).ToList();
 
            return null;
        }
 
        public float[] GetActionsAsOneHotVector(int nActionCount)
        {
            float[] rg = new float[m_rgItems.Length * nActionCount];
 
            for (int i = 0; i < m_rgItems.Length; i++)
            {
                int nAction = m_rgItems[i].Action;
 
                for (int j = 0; j < nActionCount; j++)
                {
                    rg[(i * nActionCount) + j] = (j == nAction) ? 1 : 0;
                }
            }
 
            return rg;
        }
 
        public float[] GetInvertedDoneAsOneHotVector()
        {
            float[] rgDoneInv = new float[m_rgItems.Length];
 
            for (int i = 0; i < m_rgItems.Length; i++)
            {
                if (m_rgItems[i].IsTerminated)
                    rgDoneInv[i] = 0;
                else
                    rgDoneInv[i] = 1;
            }
 
            return rgDoneInv;
        }
 
        public float[] GetRewards()
        {
            return m_rgItems.Select(p => (float)p.Reward).ToArray();
        }
 
        public void Save(string strFile)
        {
            if (File.Exists(strFile))
                File.Delete(strFile);
 
            using (StreamWriter sw = new StreamWriter(strFile))
            {
                for (int i = 0; i < Count; i++)
                {
                    MemoryItem mi = m_rgItems[i];
                    string strLine = mi.CurrentData.ToArrayAsString(4) + "," + mi.Action.ToString() + "," + mi.Reward.ToString() + "," + (mi.IsTerminated ? 1 : 0).ToString() + "," + mi.NextData.ToArrayAsString(4);
                    sw.WriteLine(strLine);
                }
            }
        }
 
        public void Load(string strFile)
        {
            m_nNextIdx = 0;
            m_nCount = 0;
 
            List<MemoryItem> rg = new List<MemoryItem>();
 
            using (StreamReader sr = new StreamReader(strFile))
            {
                string strLine = sr.ReadLine();
 
                while (strLine != null)
                {
                    string[] rgstr = strLine.Split(',');
                    int nIdx = 0;
 
                    List<double> rgdfData = new List<double>();
                    rgdfData.Add(BaseParameter.ParseDouble(rgstr[nIdx])); nIdx++;
                    rgdfData.Add(BaseParameter.ParseDouble(rgstr[nIdx])); nIdx++;
                    rgdfData.Add(BaseParameter.ParseDouble(rgstr[nIdx])); nIdx++;
                    rgdfData.Add(BaseParameter.ParseDouble(rgstr[nIdx])); nIdx++;
                    SimpleDatum sdCurrent = new SimpleDatum(true, 4, 1, 1, -1, DateTime.MinValue, rgdfData, 0, false, -1);
 
                    int nAction = int.Parse(rgstr[nIdx]); nIdx++;
                    double dfReward = BaseParameter.ParseDouble(rgstr[nIdx]); nIdx++;
                    bool bTerminated = (int.Parse(rgstr[nIdx]) == 1) ? true : false; nIdx++;
 
                    rgdfData = new List<double>();
                    rgdfData.Add(BaseParameter.ParseDouble(rgstr[nIdx])); nIdx++;
                    rgdfData.Add(BaseParameter.ParseDouble(rgstr[nIdx])); nIdx++;
                    rgdfData.Add(BaseParameter.ParseDouble(rgstr[nIdx])); nIdx++;
                    rgdfData.Add(BaseParameter.ParseDouble(rgstr[nIdx])); nIdx++;
                    SimpleDatum sdNext = new SimpleDatum(true, 4, 1, 1, -1, DateTime.MinValue, rgdfData, 0, false, -1);
 
                    rg.Add(new MemoryItem(null, sdCurrent, nAction, null, sdNext, dfReward, bTerminated, 0, 0));
                    strLine = sr.ReadLine();
                }
            }
 
            foreach (MemoryItem m in rg)
            {
                Add(m);
            }
        }
    }
 
    public class MemoryItem 
    {
        StateBase m_state0;
        StateBase m_state1;
        SimpleDatum m_x0;
        SimpleDatum m_x1;
        int m_nAction;
        int m_nIteration;
        int m_nEpisode;
        bool m_bTerminated;
        double m_dfReward;
 
        public MemoryItem(StateBase currentState, SimpleDatum currentData, int nAction, StateBase nextState, SimpleDatum nextData, double dfReward, bool bTerminated, int nIteration, int nEpisode)
        {
            m_state0 = currentState;
            m_state1 = nextState;
            m_x0 = currentData;
            m_x1 = nextData;
            m_nAction = nAction;
            m_bTerminated = bTerminated;
            m_dfReward = dfReward;
            m_nIteration = nIteration;
            m_nEpisode = nEpisode;
        }
 
        public bool IsTerminated
        {
            get { return m_bTerminated; }
        }
 
        public double Reward
        {
            get { return m_dfReward; }
            set { m_dfReward = value; }
        }
 
        public StateBase CurrentState
        {
            get { return m_state0; }
        }
 
        public StateBase NextState
        {
            get { return m_state1; }
        }
 
        public SimpleDatum CurrentData
        {
            get { return m_x0; }
        }
 
        public SimpleDatum NextData
        {
            get { return m_x1; }
        }
 
        public int Action
        {
            get { return m_nAction; }
        }
 
        public int Iteration
        {
            get { return m_nIteration; }
        }
 
        public int Episode
        {
            get { return m_nEpisode; }
        }
 
        public override string ToString()
        {
            return "episode = " + m_nEpisode.ToString() + " action = " + m_nAction.ToString() + " reward = " + m_dfReward.ToString("N2");
        }
 
        private string tostring(float[] rg)
        {
            string str = "{";
 
            for (int i = 0; i < rg.Length; i++)
            {
                str += rg[i].ToString("N5");
                str += ",";
            }
 
            str = str.TrimEnd(',');
            str += "}";
 
            return str;
        }
    }
}