MasterIndexes.cs

using MyCaffe.basecode;
using MyCaffe.basecode.descriptors;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
 
namespace MyCaffe.db.image
{
    public class MasterIndexes : IDisposable
    {
        protected SourceDescriptor m_src;
        protected Index m_index = null;
        protected LabelIndex m_rgLabels = null;
        protected Index m_boosted = null;
        protected LabelIndex m_rgLabelsBoosted = null;
        CryptoRandom m_random;
        CryptoRandom m_randomLabel = new CryptoRandom(CryptoRandom.METHOD.CRYPTO);
        CryptoRandom m_randomLabelBoosted = new CryptoRandom(CryptoRandom.METHOD.CRYPTO);
        DatasetFactory m_factory = new DatasetFactory();
        List<DbItem> m_rgImageIdx = null;
        IMGDB_SORT m_sort = IMGDB_SORT.BYIDX;
        int m_nLoadLimit = 0;
 
        public MasterIndexes(CryptoRandom random, SourceDescriptor src, int nLoadLimit = 0)
        {
            m_random = random;
            m_src = src;
            m_factory.Open(src);
            m_nLoadLimit = nLoadLimit;
 
            if (m_nLoadLimit > src.ImageCount)
                m_nLoadLimit = src.ImageCount;
 
            m_rgImageIdx = m_factory.LoadImageIndexes(false);
            load(m_rgImageIdx.Where(p => p != null).ToList());
        }
 
        private void load(List<DbItem> rgItems)
        {
            m_index = new Index("ALL", m_random, rgItems);
            List<DbItem> rgBoosted = rgItems.Where(p => p.Boost > 0).ToList();
            m_boosted = new Index("BOOSTED", m_random, rgBoosted, -1, true);
 
            List<LabelDescriptor> rgLabels = m_src.Labels.OrderBy(p => p.ActiveLabel).ToList();
 
            m_rgLabels = new LabelIndex("LABELED", m_randomLabel, m_random, m_src, false, rgItems);
            m_rgLabelsBoosted = new LabelIndex("LABELED BOOSTED", m_randomLabelBoosted, m_random, m_src, true, rgBoosted);
        }
 
        public MasterIndexes(MasterIndexes idx, IMGDB_SORT sort)
        {
            m_sort = sort;
            m_src = idx.m_src;
            m_random = idx.m_random;
            m_nLoadLimit = idx.m_nLoadLimit;
            m_factory = new DatasetFactory(idx.m_factory);
 
            m_index = idx.m_index.Clone(sort);
            m_rgLabels = new LabelIndex(idx.m_rgLabels);
 
            m_boosted = idx.m_boosted.Clone(sort);
            m_rgLabelsBoosted = new LabelIndex(idx.m_rgLabelsBoosted);
        }
 
        public void Dispose()
        {
            if (m_factory != null)
            {
                m_factory.Close();
                m_factory.Dispose();
                m_factory = null;
            }
        }
 
        public List<DbItem> RawIndexes
        {
            get { return m_rgImageIdx; }
        }
 
        public int LoadLimit
        {
            get { return m_nLoadLimit; }
        }
 
        public int LabelCount
        {
            get { return m_rgLabels.Count; }
        }
 
        public List<DbItem> FindImageIndexes(DateTime dt)
        {
            return m_index.FindImageIndexes(dt);
        }
 
        public void Reload(List<DbItem> rgItems)
        {
            load(rgItems);
        }
 
        public Index GetIndex(int? nLabel = null, bool bBoosted = false)
        {
            if (!nLabel.HasValue)
            {
                if (!bBoosted)
                    return m_index;
                else
                    return m_boosted;
            }
            else
            {
                if (!bBoosted)
                    return m_rgLabels.GetNextIndex(Index.SELECTION_TYPE.DIRECT, nLabel);
                else
                    return m_rgLabelsBoosted.GetNextIndex(Index.SELECTION_TYPE.DIRECT, nLabel);
            }
        }
 
        public void SetIndex(Index idx, int? nLabel = null, bool bBoosted = false)
        {
            if (!nLabel.HasValue)
            {
                if (!bBoosted)
                    m_index = idx;
                else
                    m_boosted = idx;
            }
            else
            {
                if (!bBoosted)
                    m_rgLabels.SetIndex(nLabel.Value, idx);
                else
                    m_rgLabelsBoosted.SetIndex(nLabel.Value, idx);
            }
        }
 
        public List<int> GetIndexes(int nStartIdx, int nQueryCount = int.MaxValue, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false)
        {
            if (m_nLoadLimit > 0)
                throw new Exception("The GetIndexes method is not valid when using LoadLimit > 0.");
 
            Index idx = GetIndex(null, nBoostVal.HasValue);
            List<DbItem> rgIdx = idx.FindImageIndexes(nStartIdx, nQueryCount, strFilterVal, nBoostVal, bBoostValIsExact);
            return rgIdx.Select(p => p.Index).ToList();
        }
 
        public List<int> GetIndexes(DateTime dtStart, int nQueryCount = int.MaxValue, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false)
        {
            if (m_nLoadLimit > 0)
                throw new Exception("The GetIndexes method is not valid when using LoadLimit > 0.");
 
            Index idx = GetIndex(null, nBoostVal.HasValue);
            List<DbItem> rgIdx = idx.FindImageIndexes(dtStart, nQueryCount, strFilterVal, nBoostVal, bBoostValIsExact);
            return rgIdx.Select(p => p.Index).ToList();
        }
 
        public virtual int? GetNextLabel(Index.SELECTION_TYPE type, bool bBoosted = false)
        {
            LabelIndex rgIdx = (bBoosted) ? m_rgLabelsBoosted : m_rgLabels;
            if (rgIdx.Count == 0)
                return null;
 
            return rgIdx.GetNextLabel(type, null, false);
        }
 
        public virtual int? GetNextImage(Index.SELECTION_TYPE type, int? nLabel = null, bool bBoosted = false, int nDirectIdx = -1)
        {
            if (m_nLoadLimit == 0)
            {
                Index idx = GetIndex(nLabel, bBoosted);
                return idx.GetNext(type);
            }
            else
            {
                if (type != Index.SELECTION_TYPE.RANDOM)
                    throw new Exception("Only RANDOM selections are valid when using LoadLimit > 0.");
 
                if (nLabel.HasValue)
                    throw new Exception("Label selections are not valid when using LoadLimit > 0");
 
                if (bBoosted != false)
                    throw new Exception("Boosted queries are not valid when using LoadLimit > 0.");
 
                if (nDirectIdx != -1)
                    throw new Exception("DirectIndex queries are not valid when using LoadLimit > 0.");
 
                return m_random.Next(m_nLoadLimit);
            }
        }
 
        public override string ToString()
        {
            string strOut = "";
 
            if (m_sort == IMGDB_SORT.NONE)
            {
                strOut = "NONE";
            }
            else
            {
                if ((m_sort & IMGDB_SORT.BYID) == IMGDB_SORT.BYID)
                    strOut += "ID";
 
                if ((m_sort & IMGDB_SORT.BYDESC) == IMGDB_SORT.BYDESC)
                {
                    if (strOut.Length > 0)
                        strOut += " | ";
                    strOut += "DESC";
                }
 
                if ((m_sort & IMGDB_SORT.BYIDX) == IMGDB_SORT.BYIDX)
                {
                    if (strOut.Length > 0)
                        strOut += " | ";
                    strOut += "IDX";
                }
 
                if ((m_sort & IMGDB_SORT.BYID_DESC) == IMGDB_SORT.BYID_DESC)
                {
                    if (strOut.Length > 0)
                        strOut += " | ";
                    strOut += "ID(desc)";
                }
 
 
                if ((m_sort & IMGDB_SORT.BYTIME) == IMGDB_SORT.BYTIME)
                {
                    if (strOut.Length > 0)
                        strOut += " | ";
                    strOut += "TIME";
                }
            }
 
            return strOut;
        }
    }
 
#pragma warning disable 1591
 
    public class LabelIndex 
    {
        string m_strName;
        CryptoRandom m_randomData;
        CryptoRandom m_randomLabel;
        SourceDescriptor m_src;
        Dictionary<int, int> m_rgLabelToIdxMap = new Dictionary<int, int>();
        Dictionary<int, int> m_rgIdxToLabelMap = new Dictionary<int, int>();
        Index[] m_rgLabels = null;
        bool m_bBoosted = false;
        List<int> m_rgIdx = new List<int>();
        int m_nIdx = 0;
        object m_objSync = new object();
 
        public LabelIndex(string strName, CryptoRandom randomLabel, CryptoRandom randomData, SourceDescriptor src, bool bBoosted, List<DbItem> rgItems)
        {
            m_strName = strName;
            m_randomLabel = randomLabel;
            m_randomData = randomData;
            m_src = src;
            m_bBoosted = bBoosted;
 
            m_rgIdx = new List<int>();
            m_rgLabelToIdxMap = new Dictionary<int, int>();
            m_rgIdxToLabelMap = new Dictionary<int, int>();
 
            List<LabelDescriptor> rgLabels = src.Labels.Where(p => p.ImageCount > 0).OrderBy(p => p.ActiveLabel).ToList();
            if (rgLabels.Count > 0)
            {
                m_rgLabels = new Index[rgLabels.Count];
 
                for (int i = 0; i < rgLabels.Count; i++)
                {
                    int nLabel = rgLabels[i].ActiveLabel;
                    List<DbItem> rgLabelList = rgItems.Where(p => p.Label == nLabel).ToList();
 
                    if (i < rgLabels.Count - 1)
                        rgItems = rgItems.Where(p => p.Label != nLabel).ToList();
 
                    m_rgLabels[i] = new Index(strName + " label " + nLabel.ToString(), randomData, rgLabelList, nLabel, false);
                    if (rgLabelList.Count > 0)
                            m_rgIdx.Add(i);
 
                    m_rgLabelToIdxMap[nLabel] = i;
                    m_rgIdxToLabelMap[i] = nLabel;
                }
            }
        }
 
        public LabelIndex(LabelIndex idx)
        {
            m_strName = idx.m_strName + " copy";
            m_randomLabel = idx.m_randomLabel;
            m_randomData = idx.m_randomData;
            m_src = idx.m_src;
            m_bBoosted = idx.m_bBoosted;
 
            m_rgIdx = new List<int>();
 
            if (idx.m_rgLabels != null && idx.m_rgLabels.Length > 0)
            {
                m_rgLabels = new Index[idx.m_rgLabels.Length];
 
                bool bFillLabelMap = false;
                if (m_rgLabelToIdxMap == null || m_rgLabelToIdxMap.Count == 0 || m_rgIdxToLabelMap == null || m_rgIdxToLabelMap.Count == 0)
                {
                    m_rgLabelToIdxMap = new Dictionary<int, int>();
                    m_rgIdxToLabelMap = new Dictionary<int, int>();
                    bFillLabelMap = true;
                }
 
                for (int i = 0; i < idx.m_rgLabels.Length; i++)
                {
                    m_rgLabels[i] = idx.m_rgLabels[i].Clone();
                    if (m_rgLabels[i].Count > 0)
                        m_rgIdx.Add(i);
 
                    if (bFillLabelMap)
                    {
                        int nLabel = m_rgLabels[i].Label;
                        m_rgLabelToIdxMap[nLabel] = i;
                        m_rgIdxToLabelMap[i] = nLabel;
                    }
                }
            }
        }
 
        public void ReLoad()
        {
            if (m_rgIdx.Count == m_rgLabels.Length)
                return;
 
            lock (m_objSync)
            {
                for (int i = 0; i < m_rgLabels.Length; i++)
                {
                    if (m_rgLabels[i].Count > 0)
                        m_rgIdx.Add(i);
                }
            }
        }
 
        public void SetIndex(int nLabel, Index idx)
        {
            int nIdx = m_rgLabelToIdxMap[nLabel];
            m_rgLabels[nIdx] = idx;
        }
 
        public int Count
        {
            get { return (m_rgLabels == null) ? 0 : m_rgLabels.Length; }
        }
 
        public bool Boosted
        {
            get { return m_bBoosted; }
        }
 
        public bool IsEmpty
        {
            get { return (m_rgIdx.Count == 0) ? true : false; }
        }
 
        public LabelIndex Clone()
        {
            return new LabelIndex(this);
        }
 
        public int? GetNextLabel(Index.SELECTION_TYPE type, int? nLabel, bool bRemove = false)
        {
            if (m_rgIdx.Count == 0)
                return null;
 
            if (nLabel.HasValue)
            {
                return nLabel.Value;
            }
            else if (type == Index.SELECTION_TYPE.SEQUENTIAL)
            {
                int nIdx = m_rgIdx[m_nIdx];
 
                m_nIdx++;
                if (m_nIdx >= m_rgIdx.Count)
                    m_nIdx = 0;
 
                return m_rgIdxToLabelMap[nIdx];
            }
            else
            {
                lock (m_objSync)
                {
                    int nIdxLoc = 0;
 
                    // Reload on the fly if empty.
                    if (m_rgIdx.Count == 0)
                    {
                        for (int i = 0; i < m_rgLabels.Length; i++)
                        {
                            if (m_rgLabels[i].Count > 0)
                                m_rgIdx.Add(i);
                        }
                    }
 
                    int nIdx = m_rgIdx[nIdxLoc];
 
                    if (m_rgIdx.Count > 1)
                    {
                        nIdxLoc = m_randomLabel.Next(0, m_rgIdx.Count - 1, true);
                        nIdx = m_rgIdx[nIdxLoc];
                    }
 
                    if (bRemove)
                        m_rgIdx.RemoveAt(nIdxLoc);
 
                    return m_rgIdxToLabelMap[nIdx];
                }
            }
        }
 
        public Index GetNextIndex(Index.SELECTION_TYPE type, int? nLabel, bool bRemove = false)
        {
            nLabel = GetNextLabel(type, nLabel, bRemove);
            if (!nLabel.HasValue)
                return null;
 
            if (!m_rgLabelToIdxMap.ContainsKey(nLabel.Value))
                return null;
            
            int nIdx = m_rgLabelToIdxMap[nLabel.Value];
 
            return m_rgLabels[nIdx];
        }
 
        public override string ToString()
        {
            return m_strName;
        }
    }
 
    public class Index 
    {
        string m_strName;
        CryptoRandom m_random;
        int m_nLabel = -1;
        bool m_bBoosted = false;
        int m_nIdx = 0;
        List<DbItem> m_rgItems;
        double m_dfProbability = 0;
        object m_objSync = new object();
 
        public enum SELECTION_TYPE
        {
            DIRECT,
            SEQUENTIAL,
            RANDOM
        }
 
        public Index(string strName, CryptoRandom random, List<DbItem> rgItems, int nLabel = -1, bool bBoosted = false, double dfProbability = 0)
        {
            m_strName = strName;
            m_random = random;
            m_rgItems = rgItems;
            m_nLabel = nLabel;
            m_bBoosted = bBoosted;
            m_dfProbability = dfProbability;
        }
 
        public int Count
        {
            get { return m_rgItems.Count; }
        }
 
        public double Probability
        {
            get { return m_dfProbability; }
            set { m_dfProbability = value; }
        }
 
        public int Label
        {
            get { return m_nLabel; }
        }
 
        public bool Boosted
        {
            get { return m_bBoosted; }
        }
 
        public bool IsEmpty
        {
            get { return (m_rgItems.Count == 0) ? true : false; }
        }
 
        public Index Clone(IMGDB_SORT sort = IMGDB_SORT.NONE)
        {
            List<DbItem> rgItems = new List<DbItem>();
 
            for (int i = 0; i < m_rgItems.Count; i++)
            {
                rgItems.Add(m_rgItems[i].Clone());
            }
 
            switch (sort)
            {
                case IMGDB_SORT.BYID:
                    rgItems = rgItems.OrderBy(p => p.ID).ToList();
                    break;
 
                case IMGDB_SORT.BYID_DESC:
                    rgItems = rgItems.OrderByDescending(p => p.ID).ToList();
                    break;
 
                case IMGDB_SORT.BYIDX:
                    rgItems = rgItems.OrderBy(p => p.Index).ToList();
                    break;
 
                case IMGDB_SORT.BYDESC:
                    rgItems = rgItems.OrderBy(p => p.Desc).ThenBy(p => p.Index).ToList();
                    break;
 
                case IMGDB_SORT.BYTIME:
                    rgItems = rgItems.OrderBy(p => p.Time).ToList();
                    break;
 
                default:
                    if ((sort & (IMGDB_SORT.BYDESC | IMGDB_SORT.BYTIME)) == (IMGDB_SORT.BYDESC | IMGDB_SORT.BYTIME))
                        rgItems = rgItems.OrderBy(p => p.Desc).ThenBy(p => p.Time).ToList();
                    break;
            }
 
            return new Index(m_strName + " copy", m_random, rgItems, m_nLabel, m_bBoosted, m_dfProbability);
        }
 
        public List<DbItem> FindImageIndexes(int nStartIdx, int nQueryCount = int.MaxValue, string strFilter = null, int? nBoostVal = null, bool bBoostValIsExact = false)
        {
            IEnumerable<DbItem> iQuery = m_rgItems.Where(p => p.Index >= nStartIdx);
 
            if (strFilter != null)
                iQuery = iQuery.Where(p => p.Desc == strFilter);
 
            if (nBoostVal.HasValue)
            {
                if (bBoostValIsExact)
                    iQuery = iQuery.Where(p => p.Boost == nBoostVal.Value);
                else
                    iQuery = iQuery.Where(p => p.Boost >= nBoostVal.Value);
            }
 
            return iQuery.Take(nQueryCount).ToList();
        }
 
        public List<DbItem> FindImageIndexes(DateTime dtStart, int nQueryCount = int.MaxValue, string strFilter = null, int? nBoostVal = null, bool bBoostValIsExact = false)
        {
            IEnumerable<DbItem> iQuery = m_rgItems.Where(p => p.Time >= dtStart);
 
            if (strFilter != null)
                iQuery = iQuery.Where(p => p.Desc == strFilter);
 
            if (nBoostVal.HasValue)
            {
                if (bBoostValIsExact)
                    iQuery = iQuery.Where(p => p.Boost == nBoostVal.Value);
                else
                    iQuery = iQuery.Where(p => p.Boost >= nBoostVal.Value);
            }
 
            return iQuery.Take(nQueryCount).ToList();
        }
 
        public List<DbItem> FindImageIndexes(DateTime dt)
        {
            return m_rgItems.Where(p => p.Time == dt).ToList();
        }
 
        public int? GetNext(SELECTION_TYPE type, bool bRemove = false)
        {
            if (m_rgItems.Count == 0)
                return null;
 
            if (type == SELECTION_TYPE.SEQUENTIAL)
            {
                int nIdx = m_rgItems[m_nIdx].Index;
 
                m_nIdx++;
                if (m_nIdx >= m_rgItems.Count)
                    m_nIdx = 0;
 
                return nIdx;
            }
            else
            {
                lock (m_objSync)
                {
                    int nCount = m_rgItems.Count;
                    if (nCount == 0)
                        return null;
 
                    int nIdx = m_random.Next(nCount);
                    if (m_rgItems[nIdx] == null)
                    {
                        m_rgItems.RemoveAt(nIdx);
                        nCount = m_rgItems.Count;
                        if (nCount == 0)
                            return null;
 
                        nIdx = m_random.Next(nCount);
                    }
 
                    int nFinalIdx = m_rgItems[nIdx].Index;
 
                    if (bRemove)
                        m_rgItems.RemoveAt(nIdx);
 
                    m_nIdx = nIdx + 1;
                    if (m_nIdx == m_rgItems.Count)
                        m_nIdx = 0;
 
                    return nFinalIdx;
                }
            }
        }
 
        public int? GetIndex(int nDirectIdx)
        {
            if (nDirectIdx < 0 || nDirectIdx >= m_rgItems.Count)
                return null;
 
            m_nIdx = nDirectIdx;
            if (m_nIdx == m_rgItems.Count)
                m_nIdx = 0;
 
            return m_rgItems[nDirectIdx].Index;
        }
 
        public override string ToString()
        {
            return m_strName + ": Count = " + m_rgItems.Count().ToString() + " CurIdx = " + m_nIdx.ToString() + "; Label = " + m_nLabel.ToString() + "; Boosted = " + m_bBoosted.ToString() + " => (" + m_rgItems.Count.ToString() + ") p = " + m_dfProbability.ToString("P");
        }
    }
 
#pragma warning restore 1591
}