MyCaffeTemporalDatabase.cs

using MyCaffe.basecode;
using MyCaffe.basecode.descriptors;
using MyCaffe.db.image;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Linq;
using System.Runtime.InteropServices;
using System.Security.Cryptography;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
 
namespace MyCaffe.db.temporal
{
    public partial class MyCaffeTemporalDatabase : Component, IXTemporalDatabaseBase
    {
        SettingsCaffe m_settings;
        PropertySet m_prop;
        CryptoRandom m_random = null;
        EventWaitHandle m_evtInitializing = null;
        EventWaitHandle m_evtInitialized = null;
        EventWaitHandle m_evtAbortInitialization = null;
        bool m_bEnabled = false;
        static object m_syncObject = new object();
        DB_ITEM_SELECTION_METHOD m_valueSelectionMethod = DB_ITEM_SELECTION_METHOD.RANDOM;
        DB_LABEL_SELECTION_METHOD m_itemSelectionMethod = DB_LABEL_SELECTION_METHOD.RANDOM;
        Log m_log;
        DB_LOAD_METHOD m_loadMethod = DB_LOAD_METHOD.LOAD_ON_DEMAND;
        int m_nLoadLimit = 0;
        Guid m_userGuid;
        DatasetCollection m_rgDataSets = new DatasetCollection();
        Dictionary<int, TemporalSet> m_rgTemporalSets = new Dictionary<int, TemporalSet>();
        
        public MyCaffeTemporalDatabase(Log log, PropertySet prop)
        {
            m_log = log;
            m_prop = prop;
            InitializeComponent();
        }
 
        public MyCaffeTemporalDatabase(IContainer container)
        {
            container.Add(this);
 
            InitializeComponent();
        }
 
        public void CleanUp(int nDsId = 0, bool bForce = false)
        {
            foreach (TemporalSet ts in m_rgTemporalSets.Values)
            {
                ts.CleanUp();
            }
 
            m_rgTemporalSets.Clear();
            m_rgDataSets.CleanUp(nDsId);
        }
 
        public static void CreateDatabaseTables(ConnectInfo ci, bool bTemporalOnly)
        {
            if (ci == null)
                ci = EntitiesConnection.GlobalDatabaseConnectInfo;
 
            DatabaseManagement dbMgr = new DatabaseManagement(ci);
            if (bTemporalOnly)
            {
                dbMgr.CreateTemporalTables();
            }
            else
            {
                Exception excpt = dbMgr.CreateDatabase();
 
                if (excpt != null)
                    throw excpt;
            }
        }
 
 
        public void Reset()
        {
            foreach (KeyValuePair<int, TemporalSet> kv in m_rgTemporalSets)
            {
                kv.Value.Reset();
            }
        }
 
        public int GetTotalSize(int nDsId, Phase phase, int nHistoricalSteps, int nFutureSteps)
        {
            DataSet ds = m_rgDataSets.Find(nDsId);
            if (ds == null)
                return 0;
 
            SourceDescriptor sd = (phase == Phase.TRAIN) ? ds.Dataset.TrainingSource : ds.Dataset.TestingSource;
 
            if (sd == null)
                return 0;
 
            if (sd.TemporalDescriptor == null)
                return 0;
 
            int nStepsPerBlock = nHistoricalSteps + nFutureSteps;
            int? nTotalSteps = sd.TemporalDescriptor.ValueItemDescriptors.Max(p => p.Steps);
            if (!nTotalSteps.HasValue || nTotalSteps.Value < nStepsPerBlock)
                m_log.FAIL("Stream: " + sd.Name + " - The number of historical and future steps must be less than the number of steps in the temporal data stream.  The steps per block = " + nStepsPerBlock.ToString() + " and the total steps = " + nTotalSteps.ToString() + " items.");
 
            int nRowBlocks = nTotalSteps.Value - nStepsPerBlock;
            int nTotal = sd.TemporalDescriptor.ValueItemDescriptors.Count * nRowBlocks;
 
            return nTotal;
        }
 
        public DatasetDescriptor GetDatasetById(int nDsId)
        {
            DataSet ds = m_rgDataSets.Find(nDsId);
            if (ds == null)
                return null;
 
            return ds.Dataset;
        }
 
        public DatasetDescriptor GetDatasetByName(string strDs)
        {
            DataSet ds = m_rgDataSets.Find(strDs);
            if (ds == null)
            {
                DatabaseLoader dbLoader = new DatabaseLoader();
                return dbLoader.LoadDatasetFromDb(strDs);
            }
 
            return ds.Dataset;
        }
 
        public int GetDatasetID(string strDs)
        {
            return m_rgDataSets.GetDatasetID(strDs);
        }
 
        public double GetDatasetLoadedPercentById(int nDatasetID, out double dfTraining, out double dfTesting)
        {
            DataSet ds = m_rgDataSets.Find(nDatasetID);
            if (ds == null)
            {
                dfTraining = 0;
                dfTesting = 0;
                return 0;
            }
 
            return ds.GetLoadPercent(out dfTraining, out dfTesting);
        }
 
        public double GetDatasetLoadedPercentByName(string strDataset, out double dfTraining, out double dfTesting)
        {
            return GetDatasetLoadedPercentById(GetDatasetID(strDataset), out dfTraining, out dfTesting);
        }
 
        public string GetDatasetName(int nDsId)
        {
            DataSet ds = m_rgDataSets.Find(nDsId);
            if (ds == null)
                return null;
 
            return ds.Dataset.Name;
        }
 
        public static Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD> GetSelectionMethod(SettingsCaffe s)
        {
            DB_ITEM_SELECTION_METHOD itemSelectionMethod = DB_ITEM_SELECTION_METHOD.NONE;
            DB_LABEL_SELECTION_METHOD valueSelectionMethod = DB_LABEL_SELECTION_METHOD.NONE;
 
            if (s.EnableRandomInputSelection)
                itemSelectionMethod |= DB_ITEM_SELECTION_METHOD.RANDOM;
 
            if (s.EnableLabelBalancing)
                valueSelectionMethod |= DB_LABEL_SELECTION_METHOD.RANDOM;
 
            return new Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD>(valueSelectionMethod, itemSelectionMethod);
        }
 
        public static Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD> GetSelectionMethod(ProjectEx p)
        {
            DB_ITEM_SELECTION_METHOD valueSelectionMethod = DB_ITEM_SELECTION_METHOD.NONE;
            DB_LABEL_SELECTION_METHOD itemSelectionMethod = DB_LABEL_SELECTION_METHOD.NONE;
 
            if (p.EnableRandomSelection)
                valueSelectionMethod |= DB_ITEM_SELECTION_METHOD.RANDOM;
 
            if (p.EnableLabelBalancing)
                itemSelectionMethod |= DB_LABEL_SELECTION_METHOD.RANDOM;
 
            return new Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD>(itemSelectionMethod, valueSelectionMethod);
        }
 
        public Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD> GetSelectionMethod()
        {
            return new Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD>(m_itemSelectionMethod, m_valueSelectionMethod);
        }
 
        public SourceDescriptor GetSourceById(int nSrcId)
        {
            return m_rgDataSets.FindSourceByID(nSrcId);
        }
 
        public SourceDescriptor GetSourceByName(string strSrc)
        {
            return m_rgDataSets.FindSourceByName(strSrc);
        }
 
        public int GetSourceID(string strSrc)
        {
            SourceDescriptor sd = GetSourceByName(strSrc);
            if (sd == null)
                return 0;
 
            return sd.ID;
        }
 
        public string GetSourceName(int nSrcId)
        {
            SourceDescriptor sd = GetSourceById(nSrcId);
            if (sd == null)
                return null;
 
            return sd.Name;
        }
 
        public bool ReloadDataset(int nDsId) 
        {
            UnloadDatasetById(nDsId);
            return InitializeWithDsId1(m_settings, nDsId);
        }
 
        public bool UnloadDatasetById(int nDsId)
        {
            if (nDsId > 0)
            {
                DataSet ds = m_rgDataSets.Find(nDsId);
                if (ds != null)
                {
                    m_rgDataSets.CleanUp(nDsId);
                    return true;
                }
            }
            else
            {
                m_rgDataSets.CleanUp(0);
            }
 
            return false;
        }
 
        public bool UnloadDatasetByName(string strDataset) 
        {
            return UnloadDatasetById(GetDatasetID(strDataset));
        }
 
        public DB_VERSION GetVersion()
        {
            return DB_VERSION.TEMPORAL;
        }
 
        public void SetInitializationProperties(PropertySet prop)
        {
            m_prop = prop;
        }
 
        public bool InitializeWithDsId1(SettingsCaffe s, int nDataSetID, string strEvtCancel = null, int nPadW = 0, int nPadH = 0, PropertySet prop = null)
        {
            m_settings = s;
            m_evtAbortInitialization = new AutoResetEvent(false);
 
            DataSet ds = m_rgDataSets.Find(nDataSetID);
            if (ds == null)
            {
                DatabaseLoader dsLoader = new DatabaseLoader();
                DatasetDescriptor dsd = dsLoader.LoadDatasetFromDb(nDataSetID);
                ds = m_rgDataSets.Add(dsd, m_log);
            }
 
            if (prop == null)
                prop = m_prop;
 
            if (prop == null)
                throw new Exception("You must first call SetInitializationProperties with the properties to use for initialization.");
 
            bool bNormalizeData = prop.GetPropertyAsBool("NormalizedData", true);
            int nHistSteps = prop.GetPropertyAsInt("HistoricalSteps", 0);
            int nFutureSteps = prop.GetPropertyAsInt("FutureSteps", 0);
            int nChunks = prop.GetPropertyAsInt("Chunks", 1024);
 
            if (nHistSteps == 0)
                throw new Exception("The historical steps are missing from the properties, please add the 'HistoricalSteps' property with a value > 0.");
            if (nFutureSteps == 0)
                throw new Exception("The future steps are missing from the properties, please add the 'FutureSteps' property with a value > 0.");
 
            if (nHistSteps < 0 || nFutureSteps < 0)
                throw new Exception("The historical and future steps must be > 0.");
 
            return ds.Load(s.DbLoadMethod, s.DbLoadLimit, s.DbAutoRefreshScheduledReplacementPercent, s.DbAutoRefreshScheduledUpdateInMs, bNormalizeData, nHistSteps, nFutureSteps, nChunks, m_evtAbortInitialization);
        }
 
        public bool InitializeWithDs1(SettingsCaffe s, DatasetDescriptor ds1, string strEvtCancel = null, PropertySet prop = null)
        {
            return InitializeWithDsId1(s, ds1.ID, strEvtCancel, 0, 0, prop);
        }
 
        public bool InitializeWithDsName1(SettingsCaffe s, string strDs, string strEvtCancel = null, PropertySet prop = null)
        {
            int nDsID = m_rgDataSets.GetDatasetID(strDs);
            if (nDsID == 0)
            {
                DatabaseTemporal db = new DatabaseTemporal();
                nDsID = db.GetDatasetID(strDs);
                if (nDsID == 0)
                    return false;
            }
 
            return InitializeWithDsId1(s, nDsID, strEvtCancel, 0, 0, null);
        }
 
        public bool LoadDatasetByID1(int nDsId, string strEvtCancel = null)
        {
            SettingsCaffe s = new SettingsCaffe();
            s.DbLoadMethod = DB_LOAD_METHOD.LOAD_ALL;
            s.DbLoadLimit = 0;
            return InitializeWithDsId1(s, nDsId, strEvtCancel);
        }
 
        public bool LoadDatasetByName1(string strDs, string strEvtCancel = null)
        {
            SettingsCaffe s = new SettingsCaffe();
            s.DbLoadMethod = DB_LOAD_METHOD.LOAD_ALL;
            s.DbLoadLimit = 0;
            return InitializeWithDsName1(s, strDs, strEvtCancel);
        }
 
        private TemporalSet getTemporalSet(int nSrcId)
        {
            TemporalSet ts = null;
 
            if (m_rgTemporalSets.ContainsKey(nSrcId))
            {
                ts = m_rgTemporalSets[nSrcId];
            }
            else
            {
                ts = m_rgDataSets.FindTemporalSetBySourceID(nSrcId);
                m_rgTemporalSets.Add(nSrcId, ts);
            }
 
            return ts;
        }
 
        public SimpleTemporalDatumCollection QueryTemporalItem(int nQueryIdx, int nSrcId, ref int? nItemIdx, ref int? nValueIdx, DB_LABEL_SELECTION_METHOD? itemSelectionOverride = null, DB_ITEM_SELECTION_METHOD? valueSelectionOverride = null, bool bEnableDebug = false, string strDebugPath = null)
        {
            TemporalSet ts = getTemporalSet(nSrcId);
            DB_LABEL_SELECTION_METHOD itemSelection = (itemSelectionOverride.HasValue) ? itemSelectionOverride.Value : m_itemSelectionMethod;
            DB_ITEM_SELECTION_METHOD valueSelection = (valueSelectionOverride.HasValue) ? valueSelectionOverride.Value : m_valueSelectionMethod;
 
            return ts.GetData(nQueryIdx, ref nItemIdx, ref nValueIdx, itemSelection, valueSelection, 1, bEnableDebug, strDebugPath);
        }
 
 
        public void SetConnection(ConnectInfo ci)
        {
            MyCaffe.db.image.EntitiesConnection.GlobalDatabaseConnectInfo = ci;
        }
 
        public void SetSelectionMethod(DB_LABEL_SELECTION_METHOD? item, DB_ITEM_SELECTION_METHOD? value)
        {
            if (item.HasValue)
                m_itemSelectionMethod = item.Value;
            if (value.HasValue)
                m_valueSelectionMethod = value.Value;
        }
 
        //---------------------------------------------------------------------
        //  Not Implemented
        //---------------------------------------------------------------------
        #region Not Implemented
        public int FindItemIndex(int nSrcId, DateTime dt, string strDescription) 
        {
            throw new NotImplementedException();
        }
 
        public SimpleDatum GetItem(int nItemID, params int[] rgSrcId) 
        {
            throw new NotImplementedException();
        }
 
        public int GetItemCount(int nSrcId, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false) 
        {
            TemporalSet ts = getTemporalSet(nSrcId);
            return ts.GetCount();
        }
 
        public SimpleDatum GetItemMean(int nSrcId) 
        {
            throw new NotImplementedException();
        }
 
        public List<SimpleDatum> GetItems(int nSrcId, int[] rgIdx, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false) 
        {
            throw new NotImplementedException();
        }
 
        public List<SimpleDatum> GetItemsFromIndex(int nSrcId, int nStartIdx, int nQueryCount = int.MaxValue, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false, bool bAttemptDirectLoad = false) 
        {
            throw new NotImplementedException();
        }
 
        public List<SimpleDatum> GetItemsFromTime(int nSrcId, DateTime dtStart, int nQueryCount = int.MaxValue, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false) 
        {
            throw new NotImplementedException();
        }
 
        public bool GetLoadItemDataCriteria() 
        {
            throw new NotImplementedException();
        }
 
        public bool GetLoadItemDebugData() 
        {
            throw new NotImplementedException();
        }
 
        public SimpleDatum QueryItem(int nSrcId, int nIdx, DB_LABEL_SELECTION_METHOD? labelSelectionOverride = null, DB_ITEM_SELECTION_METHOD? imageSelectionOverride = null, int? nLabel = null, bool bLoadDataCriteria = false, bool bLoadDebugData = false) 
        {
            return null;
        }
 
        public SimpleDatum QueryItemMean(int nSrcId) 
        {
            return null;
        }
 
        public SimpleDatum QueryItemMeanFromDataset(int nDatasetId) 
        {
            return null;
        }
 
        public SimpleDatum QueryItemMeanFromDb(int nSrcId) 
        {
            return null;
        }
 
        #endregion
    }
}