MyCaffeImageDatabase2.cs

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Drawing;
using System.Threading;
using MyCaffe.basecode;
using MyCaffe.basecode.descriptors;
 
namespace MyCaffe.db.image
{
    public partial class MyCaffeImageDatabase2 : Component, IXImageDatabase2
    {
        CryptoRandom m_random = null;
        DatasetFactory m_factory;
        string m_strID = "";
        int m_nStrIDHashCode = 0;
        EventWaitHandle m_evtInitializing = null;
        EventWaitHandle m_evtInitialized = null;
        EventWaitHandle m_evtAbortInitialization = null;
        bool m_bEnabled = false;
        static object m_syncObject = new object();
        static Dictionary<int, DatasetExCollection2> m_colDatasets = new Dictionary<int, DatasetExCollection2>();
        static Dictionary<int, LabelMappingCollection> m_rgLabelMappings = new Dictionary<int, LabelMappingCollection>();
        Dictionary<int, SimpleDatum> m_rgMeanCache = new Dictionary<int, SimpleDatum>();
        double m_dfSuperBoostProbability = 0;
        DB_ITEM_SELECTION_METHOD m_imageSelectionMethod = DB_ITEM_SELECTION_METHOD.RANDOM;
        DB_LABEL_SELECTION_METHOD m_labelSelectionMethod = DB_LABEL_SELECTION_METHOD.RANDOM;
        Log m_log;
        DB_LOAD_METHOD m_loadMethod = DB_LOAD_METHOD.LOAD_ON_DEMAND;
        int m_nLoadLimit = 0;
        bool m_bSkipMeanCheck = false;
        int m_nPadW = 0;
        int m_nPadH = 0;
        Guid m_userGuid;
 
 
        public event EventHandler<CalculateImageMeanArgs> OnCalculateImageMean;
 
        public MyCaffeImageDatabase2(Log log = null, string strId = "default", int nSeed = 0)
        {
            m_factory = new DatasetFactory();
            m_userGuid = Guid.NewGuid();
            m_log = log;
            InitializeComponent();
            init(strId, nSeed);
 
            if (log != null)
                log.WriteLine("INFO: Using MyCaffe Image Database VERSION 2.");
        }
 
        public MyCaffeImageDatabase2(IContainer container)
        {
            container.Add(this);
 
            InitializeComponent();
            init();
        }
 
        public DB_VERSION GetVersion()
        {
            return DB_VERSION.IMG_V2;
        }
 
        #region Initialization and Cleanup
 
        public void SetConnection(ConnectInfo ci)
        {
            MyCaffe.db.image.EntitiesConnection.GlobalDatabaseConnectInfo = ci;
        }
 
        public long InitializeWithDsName(SettingsCaffe s, string strDs, string strEvtCancel = null)
        {
            return InitializeWithDs(s, new DatasetDescriptor(strDs), strEvtCancel);
        }
 
        public long InitializeWithDs(SettingsCaffe s, DatasetDescriptor ds, string strEvtCancel = null)
        {
            string strDsName = ds.Name;
 
            if (String.IsNullOrEmpty(strDsName))
            {
                strDsName = m_factory.FindDatasetNameFromSourceName(ds.TrainingSourceName, ds.TestingSourceName);
                if (strDsName == null)
                    throw new Exception("Could not find the dataset! You must specify either the Datast name or at least the training or testing source name!");
            }
 
            int nDsId = m_factory.GetDatasetID(strDsName);
 
            return InitializeWithDsId(s, nDsId, strEvtCancel);
        }
 
        public long InitializeWithDsId(SettingsCaffe s, int nDataSetID, string strEvtCancel = null, int nPadW = 0, int nPadH = 0)
        {
            Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD> selMethod = GetSelectionMethod(s);
 
            m_nPadW = nPadW;
            m_nPadH = nPadH;
            m_labelSelectionMethod = selMethod.Item1;
            m_imageSelectionMethod = selMethod.Item2;
            m_dfSuperBoostProbability = s.SuperBoostProbability;
            m_loadMethod = s.DbLoadMethod;
            m_nLoadLimit = s.DbLoadLimit;
            m_bSkipMeanCheck = s.SkipMeanCheck;
 
            if (m_loadMethod == DB_LOAD_METHOD.LOAD_EXTERNAL)
                m_loadMethod = DB_LOAD_METHOD.LOAD_ON_DEMAND_BACKGROUND;
 
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtInitialized, m_evtInitializing, m_evtAbortInitialization }, 0);
 
            lock (m_syncObject)
            {
                if (nWait != WaitHandle.WaitTimeout)
                {
                    if (nWait == 0)     // already initialized.
                    {
                        if (m_colDatasets.ContainsKey(m_nStrIDHashCode))
                        {
                            DatasetExCollection2 col = m_colDatasets[m_nStrIDHashCode];
 
                            if (m_log != null)
                                m_log.WriteLine("The MyCaffe Image Database is already initialized.");
 
                            DatasetEx2 ds = col.FindDataset(nDataSetID);
                            if (ds != null)
                            {
                                ds.AddUser(m_userGuid);
                                return ds.DefaultQueryState;
                            }
                        }
                    }
                    else
                    {
                        return 0;
                    }
                }
 
                try
                {
                    m_evtInitializing.Set();
 
                    m_factory.SetLoadingParameters(s.ItemDbLoadDataCriteria, s.ItemDbLoadDebugData);
 
                    DatasetDescriptor ds = m_factory.LoadDataset(nDataSetID);
                    if (ds == null)
                        throw new Exception("Could not find dataset with ID = " + nDataSetID.ToString());
 
                    List<WaitHandle> rgAbort = new List<WaitHandle>() { m_evtAbortInitialization };
                    CancelEvent evtCancel;
 
                    if (strEvtCancel != null)
                    {
                        evtCancel = new CancelEvent(strEvtCancel);
                        rgAbort.AddRange(evtCancel.Handles);
                    }
 
                    DatasetExCollection2 col = null;
 
                    if (m_colDatasets.ContainsKey(m_nStrIDHashCode))
                        col = m_colDatasets[m_nStrIDHashCode];
                    else
                        col = new DatasetExCollection2();
 
                    if (m_evtAbortInitialization.WaitOne(0))
                    {
                        col.Dispose();
                        return 0;
                    }
 
                    DatasetEx2 ds0 = new DatasetEx2(m_userGuid, m_factory, m_random);
 
                    if (OnCalculateImageMean != null)
                        ds0.OnCalculateImageMean += OnCalculateImageMean;
 
                    if (m_log != null)
                        m_log.WriteLine("Loading dataset '" + ds.Name + "'...", true);
 
                    long lQueryHandle = ds0.Initialize(ds, rgAbort.ToArray(), nPadW, nPadH, m_log, m_loadMethod, m_bSkipMeanCheck, m_nLoadLimit, s.DbAutoRefreshScheduledUpdateInMs, s.DbAutoRefreshScheduledReplacementPercent, s.VeriyDatasetOnLoad);
                    if (lQueryHandle == 0)
                    {
                        col.Dispose();
                        return 0;
                    }
 
                    if (m_log != null)
                        m_log.WriteLine("Dataset '" + ds.Name + "' loaded.", true);
 
                    col.Add(ds0);
 
                    if (!m_colDatasets.ContainsKey(m_nStrIDHashCode))
                        m_colDatasets.Add(m_nStrIDHashCode, col);
 
                    UseTrainingImagesForTesting = s.UseTrainingSourceForTesting;
 
                    return lQueryHandle;
                }
                finally
                {
                    m_evtInitialized.Set();
                    m_evtInitializing.Reset();
                }
            }
        }
 
        public bool InitializeWithDsName1(SettingsCaffe s, string strDs, string strEvtCancel = null, PropertySet prop = null)
        {
            long lHandle = InitializeWithDsName(s, strDs, strEvtCancel);
            if (lHandle == 0)
                return false;
 
            return true;
        }
 
        public bool InitializeWithDs1(SettingsCaffe s, DatasetDescriptor ds, string strEvtCancel = null, PropertySet prop = null)
        {
            long lHandle = InitializeWithDs(s, ds, strEvtCancel);
            if (lHandle == 0)
                return false;
 
            return true;
        }
 
        public bool InitializeWithDsId1(SettingsCaffe s, int nDataSetID, string strEvtCancel = null, int nPadW = 0, int nPadH = 0, PropertySet prop = null)
        {
            long lHandle = InitializeWithDsId(s, nDataSetID, strEvtCancel, nPadW, nPadH);
            if (lHandle == 0)
                return false;
 
            return true;
        }
 
        private void init(string strId = "", int nSeed = 0)
        {
            int nProcessID = Process.GetCurrentProcess().Id;
 
            m_random = new CryptoRandom(CryptoRandom.METHOD.DEFAULT, nSeed);
            m_strID = strId;
            m_nStrIDHashCode = strId.GetHashCode();
            m_evtInitializing = new EventWaitHandle(false, EventResetMode.ManualReset, "__CAFFE_IMAGEDATABASE__INITIALIZING__" + nProcessID.ToString());
            m_evtInitialized = new EventWaitHandle(false, EventResetMode.ManualReset, "__CAFFE_IMAGEDATABASE__INITIALIZED__" + nProcessID.ToString());
            m_evtAbortInitialization = new EventWaitHandle(false, EventResetMode.ManualReset, "__CAFFE_IMAGEDATABASE__ABORT_INITIALIZE__" + nProcessID.ToString());
        }
 
        public void CleanUp(int nDsId = 0, bool bForce = false)
        {
            lock (m_syncObject)
            {
                if (m_evtInitializing != null && m_evtInitializing.WaitOne(0))
                {
                    m_evtAbortInitialization.Set();
                    return;
                }
 
                List<int> rgRemove = new List<int>();
 
                foreach (KeyValuePair<int, DatasetExCollection2> col in m_colDatasets)
                {
                    DatasetExCollection2 colDs = col.Value;
 
                    if (colDs.RemoveUser(m_userGuid) || bForce)
                    {
                        rgRemove.Add(col.Key);
                        colDs.Dispose();
                    }
                }
 
                foreach (int nKey in rgRemove)
                {
                    m_colDatasets.Remove(nKey);
                }
 
                if (m_colDatasets.Count == 0)
                {
                    if (m_evtInitialized.WaitOne(0))
                        m_evtInitialized.Reset();
                }
            }
        }
 
        private void dispose()
        {
            CleanUp(0, false);
 
            if (m_evtInitialized != null)
            {
                m_evtInitialized.Dispose();
                m_evtInitialized = null;
            }
 
            if (m_evtInitializing != null)
            {
                m_evtInitializing.Dispose();
                m_evtInitializing = null;
            }
 
            if (m_evtAbortInitialization != null)
            {
                m_evtAbortInitialization.Dispose();
                m_evtAbortInitialization = null;
            }
 
            if (m_random != null)
            {
                m_random.Dispose();
                m_random = null;
            }
        }
 
        public bool WaitForDatasetToLoad(int nDsId, bool bTraining, bool bTesting, int nWait = int.MaxValue)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].WaitForDatasetToLoad(nDsId, bTraining, bTesting, nWait);
        }
 
        public bool WaitForDatasetToLoad(string strDs, bool bTraining, bool bTesting, int nWait = int.MaxValue)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].WaitForDatasetToLoad(strDs, bTraining, bTesting, nWait);
        }
 
        public bool ReloadIndexing(int nDsId)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            m_colDatasets[m_nStrIDHashCode].ReloadIndexing(nDsId);
            return true;
        }
 
        public bool ReloadIndexing(string strDs)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            m_colDatasets[m_nStrIDHashCode].ReloadIndexing(strDs);
            return true;
        }
 
        public static void CreateDatabase(ConnectInfo ci, string strPath)
        {
            if (ci == null)
                ci = EntitiesConnection.GlobalDatabaseConnectInfo;
 
            DatabaseManagement dbMgr = new DatabaseManagement(ci, strPath);
            Exception excpt = dbMgr.CreateDatabase();
 
            if (excpt != null)
                throw excpt;
        }
 
        #endregion // Initialization and Cleanup
 
        #region Refreshing
 
        public bool WaitForDatasetToRefresh(int nDsId, bool bTraining, bool bTesting, int nWait = int.MaxValue)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].WaitForDatasetToRefresh(nDsId, bTraining, bTesting, nWait);
        }
 
        public bool WaitForDatasetToRefresh(string strDs, bool bTraining, bool bTesting, int nWait = int.MaxValue)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].WaitForDatasetToRefresh(strDs, bTraining, bTesting, nWait);
        }
 
        public bool IsRefreshRunning(int nDsId, bool bTraining, bool bTesting)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].IsRefreshRunning(nDsId, bTraining, bTesting);
        }
 
        public bool IsRefreshRunning(string strDs, bool bTraining, bool bTesting)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].IsRefreshRunning(strDs, bTraining, bTesting);
        }
 
        public bool StartRefresh(string strDs, bool bTraining, bool bTesting, double dfReplacementPct)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].StartRefresh(strDs, bTraining, bTesting, dfReplacementPct);
        }
 
        public bool StopRefresh(string strDs, bool bTraining, bool bTesting)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].StopRefresh(strDs, bTraining, bTesting);
        }
 
        public bool StartRefresh(int nDsID, bool bTraining, bool bTesting, double dfReplacementPct)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].StartRefresh(nDsID, bTraining, bTesting, dfReplacementPct);
        }
 
        public bool StopRefresh(int nDsID, bool bTraining, bool bTesting)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].StopRefresh(nDsID, bTraining, bTesting);
        }
 
        public bool StartAutomaticRefreshSchedule(string strDs, bool bTraining, bool bTesting, int nPeriodInMs, double dfReplacementPct)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            if (nPeriodInMs <= 0 || dfReplacementPct <= 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].StartAutomaticRefreshSchedule(strDs, bTraining, bTesting, nPeriodInMs, dfReplacementPct);
        }
 
        public bool StopAutomaticRefreshSchedule(string strDs, bool bTraining, bool bTesting)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].StopAutomaticRefreshSchedule(strDs, bTraining, bTesting);
        }
 
        public bool GetScheduledAutoRefreshInformation(string strDs, out int nPeriodInMs, out double dfReplacementPct, out int nTrainingRefreshCount, out int nTestingRefreshCount)
        {
            nPeriodInMs = 0;
            dfReplacementPct = 0;
            nTrainingRefreshCount = 0;
            nTestingRefreshCount = 0;
 
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].GetScheduledAutoRefreshInformation(strDs, out nPeriodInMs, out dfReplacementPct, out nTrainingRefreshCount, out nTestingRefreshCount);
        }
 
        public bool StartAutomaticRefreshSchedule(int nDsID, bool bTraining, bool bTesting, int nPeriodInMs, double dfReplacementPct)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            if (nPeriodInMs <= 0 || dfReplacementPct <= 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].StartAutomaticRefreshSchedule(nDsID, bTraining, bTesting, nPeriodInMs, dfReplacementPct);
        }
 
        public bool StopAutomaticRefreshSchedule(int nDsID, bool bTraining, bool bTesting)
        {
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].StopAutomaticRefreshSchedule(nDsID, bTraining, bTesting);
        }
 
        public bool GetScheduledAutoRefreshInformation(int nDsID, out int nPeriodInMs, out double dfReplacementPct, out int nTrainingRefreshCount, out int nTestingRefreshCount)
        {
            nPeriodInMs = 0;
            dfReplacementPct = 0;
            nTrainingRefreshCount = 0;
            nTestingRefreshCount = 0;
 
            int nWait1 = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait1 == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].GetScheduledAutoRefreshInformation(nDsID, out nPeriodInMs, out dfReplacementPct, out nTrainingRefreshCount, out nTestingRefreshCount);
        }
 
        #endregion
 
        #region Query States
 
        public long CreateQueryState(int nDsId, bool bUseUniqueLabelIndexes = true, bool bUseUniqueImageIndexes = true, IMGDB_SORT sort = IMGDB_SORT.NONE)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return 0;
 
            return m_colDatasets[m_nStrIDHashCode].CreateQueryState(nDsId, bUseUniqueLabelIndexes, bUseUniqueImageIndexes, sort);
        }
 
        public long CreateQueryState(string strDs, bool bUseUniqueLabelIndexes = true, bool bUseUniqueImageIndexes = true, IMGDB_SORT sort = IMGDB_SORT.NONE)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return 0;
 
            return m_colDatasets[m_nStrIDHashCode].CreateQueryState(strDs, bUseUniqueLabelIndexes, bUseUniqueImageIndexes, sort);
        }
 
        public bool SetDefaultQueryState(int nDsId, long lQueryState)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].SetDefaultQueryState(nDsId, lQueryState);
        }
 
        public bool SetDefaultQueryState(string strDs, long lQueryState)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].SetDefaultQueryState(strDs, lQueryState);
        }
 
        public bool FreeQueryState(int nDsId, long lHandle)
        {
            return m_colDatasets[m_nStrIDHashCode].FreeQueryState(nDsId, lHandle);
        }
 
        public bool FreeQueryState(string strDs, long lHandle)
        {
            return m_colDatasets[m_nStrIDHashCode].FreeQueryState(strDs, lHandle);
        }
 
        public string GetBoostQueryHitPercentsAsTextFromSourceName(long lQueryState, string strSource)
        {           
            return m_colDatasets[m_nStrIDHashCode].FindQueryState(lQueryState, strSource).GetQueryBoostHitPercentsAsText();
        }
 
        public string GetLabelQueryHitPercentsAsTextFromSourceName(long lQueryState, string strSource)
        {
            return m_colDatasets[m_nStrIDHashCode].FindQueryState(lQueryState, strSource).GetQueryLabelHitPercentsAsText();
        }
 
        public string GetLabelQueryEpocsAsTextFromSourceName(long lQueryState, string strSource)
        {
            return m_colDatasets[m_nStrIDHashCode].FindQueryState(lQueryState, strSource).GetQueryLabelEpochsAsText();
        }
 
        public string GetBoostQueryHitPercentsAsTextFromSourceName(string strSource)
        {
            return GetBoostQueryHitPercentsAsTextFromSourceName(0, strSource);
        }
 
        public string GetLabelQueryHitPercentsAsTextFromSourceName(string strSource)
        {
            return GetLabelQueryHitPercentsAsTextFromSourceName(0, strSource);
        }
 
        public string GetLabelQueryEpocsAsTextFromSourceName(string strSource)
        {
            return GetLabelQueryEpocsAsTextFromSourceName(0, strSource);
        }
 
        #endregion // Query States
 
        #region Properties
 
        public Log OutputLog
        {
            get { return m_log; }
            set { m_log = value; }
        }
 
        public bool GetLoadItemDataCriteria()
        {
            return m_factory.LoadDataCriteria;
        }
 
        public bool GetLoadItemDebugData()
        {
            return m_factory.LoadDebugData;
        }
 
        public int GetImageCount(int nSrcId)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
 
            if (nWait == 0)
                return 0;
 
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetTotalCount();
        }
 
        public int GetItemCount(int nSrcId, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false)
        {
            return GetImageCount(0, nSrcId, strFilterVal, nBoostVal, bBoostValIsExact);
        }
 
        public int GetImageCount(long lQueryState, int nSrcId, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return 0;
 
            QueryState qstate = m_colDatasets[m_nStrIDHashCode].FindQueryState(lQueryState, nSrcId);
            ImageSet2 imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId);
 
            return imgSet.GetCount(qstate, strFilterVal, nBoostVal, bBoostValIsExact);
        }
 
        public static Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD> GetSelectionMethod(SettingsCaffe s)
        {
            DB_ITEM_SELECTION_METHOD imageSelectionMethod = DB_ITEM_SELECTION_METHOD.NONE;
            DB_LABEL_SELECTION_METHOD labelSelectionMethod = DB_LABEL_SELECTION_METHOD.NONE;
 
            if (s.EnableRandomInputSelection)
                imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.RANDOM;
 
            if (s.SuperBoostProbability > 0)
                imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.BOOST;
 
            if (s.EnableLabelBalancing)
            {
                labelSelectionMethod |= DB_LABEL_SELECTION_METHOD.RANDOM;
 
                if (s.EnableLabelBoosting)
                    labelSelectionMethod |= DB_LABEL_SELECTION_METHOD.BOOST;
            }
            else
            {
                if (s.EnablePairInputSelection)
                    imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.PAIR;
            }
 
            return new Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD>(labelSelectionMethod, imageSelectionMethod);
        }
 
        public static Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD> GetSelectionMethod(ProjectEx p)
        {
            DB_ITEM_SELECTION_METHOD imageSelectionMethod = DB_ITEM_SELECTION_METHOD.NONE;
            DB_LABEL_SELECTION_METHOD labelSelectionMethod = DB_LABEL_SELECTION_METHOD.NONE;
 
            if (p.EnableRandomSelection)
                imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.RANDOM;
 
            if (p.EnableLabelBalancing)
            {
                labelSelectionMethod |= DB_LABEL_SELECTION_METHOD.RANDOM;
 
                if (p.EnableLabelBoosting)
                    labelSelectionMethod |= DB_LABEL_SELECTION_METHOD.BOOST;
            }
            else
            {
                if (p.EnablePairSelection)
                    imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.PAIR;
            }
 
            return new Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD>(labelSelectionMethod, imageSelectionMethod);
        }
 
        public Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD> GetSelectionMethod()
        {
            return new Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD>(m_labelSelectionMethod, m_imageSelectionMethod);
        }
 
        public void SetSelectionMethod(DB_LABEL_SELECTION_METHOD? lbl, DB_ITEM_SELECTION_METHOD? img)
        {
            if (lbl.HasValue)
                m_labelSelectionMethod = lbl.Value;
 
            if (img.HasValue)
                m_imageSelectionMethod = img.Value;
        }
 
        public bool UseTrainingImagesForTesting
        {
            get { return m_colDatasets[m_nStrIDHashCode].UseTrainingSourcesForTesting; }
            set { m_colDatasets[m_nStrIDHashCode].EnableUsingTrainingSourcesForTesting(value); }
        }
 
        public double SuperBoostProbability
        {
            get { return m_dfSuperBoostProbability; }
            set { m_dfSuperBoostProbability = value; }
        }
 
        public bool SelectFromBoostOnly
        {
            get
            {
                if (m_dfSuperBoostProbability > 0)
                {
                    double dfRandom = m_random.NextDouble(0, 1);
 
                    if (dfRandom <= m_dfSuperBoostProbability)
                        return true;
                }
 
                return false;
            }
        }
 
        public bool IsEnabled
        {
            get { return m_bEnabled; }
        }
 
        public void Enable(bool bEnable)
        {
            m_bEnabled = bEnable;
        }
 
        public bool IsInitialized
        {
            get { return m_evtInitialized.WaitOne(0); }
        }
 
        #endregion // Properties
 
        #region Image Acquisition
 
        public List<SimpleDatum> GetItemsFromIndex(int nSrcId, int nStartIdx, int nQueryCount = int.MaxValue, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false, bool bAttemptDirectLoad = false)
        {
            return GetImagesFromIndex(0, nSrcId, nStartIdx, nQueryCount, strFilterVal, nBoostVal, bBoostValIsExact, bAttemptDirectLoad);
        }
 
        public List<SimpleDatum> GetItemsFromTime(int nSrcId, DateTime dtStart, int nQueryCount = int.MaxValue, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false)
        {
            return GetImagesFromTime(0, nSrcId, dtStart, nQueryCount, strFilterVal, nBoostVal, bBoostValIsExact);
        }
 
        public List<SimpleDatum> GetImagesFromIndex(long lQueryState, int nSrcId, int nStartIdx, int nQueryCount = int.MaxValue, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false, bool bAttemptDirectLoad = false)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return null;
 
            QueryState qstate = m_colDatasets[m_nStrIDHashCode].FindQueryState(lQueryState, nSrcId);
            ImageSet2 imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId);
 
            return imgSet.GetImages(qstate, nStartIdx, nQueryCount, strFilterVal, nBoostVal, bBoostValIsExact, bAttemptDirectLoad);
        }
 
        public List<SimpleDatum> GetImagesFromTime(long lQueryState, int nSrcId, DateTime dtStart, int nQueryCount = int.MaxValue, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return null;
 
            QueryState qstate = m_colDatasets[m_nStrIDHashCode].FindQueryState(lQueryState, nSrcId);
            ImageSet2 imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId);
 
            return imgSet.GetImages(qstate, dtStart, nQueryCount, strFilterVal, nBoostVal, bBoostValIsExact);
        }
 
        public List<SimpleDatum> GetItems(int nSrcId, int[] rgIdx, string strFilterVal, int? nBoostVal, bool bBoostValIsExact = false)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return null;
 
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetImages(nBoostVal.HasValue, strFilterVal, nBoostVal, rgIdx);
        }
 
        public SimpleDatum QueryImage(long lQueryState, int nSrcId, int nIdx, DB_LABEL_SELECTION_METHOD? labelSelectionOverride = null, DB_ITEM_SELECTION_METHOD? imageSelectionOverride = null, int? nLabel = null, bool bLoadDataCriteria = false, bool bLoadDebugData = false)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return null;
 
            DB_LABEL_SELECTION_METHOD labelSelectionMethod = m_labelSelectionMethod;
            DB_ITEM_SELECTION_METHOD imageSelectionMethod = m_imageSelectionMethod;
 
            if (labelSelectionOverride.HasValue)
                labelSelectionMethod = labelSelectionOverride.Value;
 
            if (imageSelectionOverride.HasValue)
                imageSelectionMethod = imageSelectionOverride.Value;
 
            if (SelectFromBoostOnly)
                imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.BOOST;
 
            QueryState qstate = m_colDatasets[m_nStrIDHashCode].FindQueryState(lQueryState, nSrcId);
            ImageSet2 imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId);
 
            return imgSet.GetImage(qstate, labelSelectionMethod, imageSelectionMethod, m_log, nLabel, nIdx, bLoadDataCriteria, bLoadDataCriteria);
        }
 
        public SimpleDatum QueryItem(int nSrcId, int nIdx, DB_LABEL_SELECTION_METHOD? labelSelectionOverride = null, DB_ITEM_SELECTION_METHOD? imageSelectionOverride = null, int? nLabel = null, bool bLoadDataCriteria = false, bool bLoadDebugData = false)
        {
            return QueryImage(0, nSrcId, nIdx, labelSelectionOverride, imageSelectionOverride, nLabel, bLoadDataCriteria, bLoadDebugData);
        }
 
        public SimpleDatum GetItem(int nImageID, params int[] rgSrcId)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
 
            if (nWait == 0)
                return null;
 
            foreach (int nSrcId in rgSrcId)
            {
                ImageSet2 imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId);
                SimpleDatum sd = imgSet.GetImage(nImageID);
 
                if (sd != null)
                    return sd;
            }
 
            return null;
        }
 
        public int FindItemIndex(int nSrcId, DateTime dt, string strDescription)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).FindImageIndex(dt, strDescription);
        }
 
        #endregion // Image Acquisition
 
        #region Image Mean
 
        public SimpleDatum GetItemMean(int nSrcId)
        {
            if (m_evtAbortInitialization.WaitOne(0))
                return null;
 
            if (!m_evtInitialized.WaitOne(0))
            {
                if (m_rgMeanCache.Keys.Contains(nSrcId))
                    return m_rgMeanCache[nSrcId];
            }
 
            SimpleDatum sd = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetImageMean(null, null, true);
 
            if (!m_rgMeanCache.ContainsKey(nSrcId))
                m_rgMeanCache.Add(nSrcId, sd);
            else
                m_rgMeanCache[nSrcId] = sd;
 
            return sd;
        }
 
        public void SetImageMean(int nSrcId, SimpleDatum d)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
 
            if (nWait == 0)
                return;
 
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).SetImageMean(d);
        }
 
        public SimpleDatum QueryItemMeanFromDataset(int nDatasetId)
        {
            DatasetEx2 ds = m_colDatasets[m_nStrIDHashCode].FindDataset(nDatasetId);
            if (ds == null)
                return null;
 
            return QueryItemMean(ds.Descriptor.TrainingSource.ID);
        }
 
        public SimpleDatum QueryItemMeanFromDb(int nSrcId)
        {
            SimpleDatum sd = QueryItemMean(nSrcId);
 
            if (sd != null)
                return sd;
 
            sd = GetItemMean(nSrcId);
            SaveImageMean(nSrcId, sd, false);
 
            return sd;
        }
 
        public void SaveImageMean(int nSrcId, SimpleDatum d, bool bUpdate)
        {
            if (m_colDatasets.ContainsKey(m_nStrIDHashCode))
                m_colDatasets[m_nStrIDHashCode].SaveImageMean(nSrcId, d, bUpdate);
        }
 
        public SimpleDatum QueryItemMean(int nSrcId)
        {
            if (!m_colDatasets.ContainsKey(m_nStrIDHashCode))
                return null;
 
            return m_colDatasets[m_nStrIDHashCode].QueryImageMean(nSrcId);
        }
 
        public bool DoesImageMeanExists(int nSrcId)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Source> rgSrc = entities.Sources.Where(p => p.ID == nSrcId).ToList();
                if (rgSrc.Count == 0)
                    return false;
 
                IQueryable<RawImageMean> iQuery = entities.RawImageMeans.Where(p => p.SourceID == nSrcId);
                if (iQuery != null)
                {
                    List<RawImageMean> rgMean = iQuery.ToList();
                    if (rgMean.Count > 0)
                        return true;
                }
 
                return false;
            }
        }
 
        #endregion // Image Mean
 
        #region Datasets
 
        public long LoadDatasetByName(string strDs, string strEvtCancel = null)
        {
            int nDsId = m_factory.GetDatasetID(strDs);
            return LoadDatasetByID(nDsId, strEvtCancel);
        }
 
        public long LoadDatasetByID(int nDsId, string strEvtCancel = null)
        {
            if (!m_evtInitialized.WaitOne(0))
                throw new Exception("The image database must be initialized first before a secondary dataset can be loaded.");
 
            if (m_evtInitializing.WaitOne(0))
                throw new Exception("The image database is in the process of being initialized.");
 
            DatasetEx2 ds = m_colDatasets[m_nStrIDHashCode].FindDataset(nDsId);
            if (ds != null)
                return 0;
 
            DatasetDescriptor desc = m_factory.LoadDataset(nDsId);
            if (desc == null)
                throw new Exception("Could not find dataset with ID = " + nDsId.ToString());
 
            if (!m_colDatasets.ContainsKey(m_nStrIDHashCode))
                throw new Exception("The image database was not initialized properly.");
 
            DatasetExCollection2 col = m_colDatasets[m_nStrIDHashCode];
            DatasetEx2 ds0 = new DatasetEx2(m_userGuid, m_factory, m_random);
 
            if (OnCalculateImageMean != null)
                ds0.OnCalculateImageMean += OnCalculateImageMean;
 
            if (m_log != null)
                m_log.WriteLine("Loading dataset '" + desc.Name + "'...");
 
            CancelEvent evtCancel = new CancelEvent(strEvtCancel);
            List<WaitHandle> rgAbort = new List<WaitHandle>(evtCancel.Handles);
 
            long lQueryState = ds0.Initialize(desc, rgAbort.ToArray(), 0, 0, m_log, m_loadMethod, false, m_nLoadLimit);
            if (lQueryState == 0)
            {
                col.Dispose();
                return 0;
            }
 
            if (m_log != null)
                m_log.WriteLine("Dataset '" + desc.Name + "' loaded.");
 
            col.Add(ds0);
 
            return lQueryState;
        }
 
        public bool LoadDatasetByID1(int nDsId, string strEvtCancel = null)
        {
            long lHandle = LoadDatasetByID(nDsId, strEvtCancel);
            if (lHandle == 0)
                return false;
 
            return true;
        }
 
        public bool LoadDatasetByName1(string strDs, string strEvtCancel = null)
        {
            long lHandle = LoadDatasetByName(strDs, strEvtCancel);
            if (lHandle == 0)
                return false;
 
            return true;
        }
 
        public bool ReloadDataset(int nDsId)
        {
            DatasetEx2 ds = m_colDatasets[m_nStrIDHashCode].FindDataset(nDsId);
            if (ds != null)
            {
                ds.ResetLabels();
                return true;
            }
 
            return false;
        }
 
        public bool UnloadDatasetByName(string strDataset)
        {
            bool bRemoved = false;
 
            lock (m_syncObject)
            {
                if (m_colDatasets.ContainsKey(m_nStrIDHashCode))
                {
                    DatasetExCollection2 col = m_colDatasets[m_nStrIDHashCode];
                    DatasetEx2 ds = col.FindDataset(strDataset);
                    if (ds != null)
                    {
                        if (m_log != null)
                            m_log.WriteLine("Unloading dataset '" + ds.DatasetName + "'.");
 
                        ds.Unload(false);
                        GC.Collect();
                        bRemoved = true;
                    }
                }
            }
 
            return bRemoved;
        }
 
        public bool UnloadDatasetById(int nDataSetID)
        {
            bool bRemoved = false;
 
            lock (m_syncObject)
            {
                if (m_colDatasets.ContainsKey(m_nStrIDHashCode))
                {
                    DatasetExCollection2 col = m_colDatasets[m_nStrIDHashCode];
 
                    foreach (DatasetEx2 ds in col)
                    {
                        if (ds != null)
                        {
                            if (ds.DatasetID == nDataSetID || nDataSetID == -1)
                            {
                                if (m_log != null)
                                    m_log.WriteLine("Unloading dataset '" + ds.DatasetName + "'.");
 
                                ds.Unload(false);
                                GC.Collect();
                                bRemoved = true;
                            }
                        }
                    }
                }
            }
 
            return bRemoved;
        }
 
        public double GetDatasetLoadedPercentByName(string strDataset, out double dfTraining, out double dfTesting)
        {
            dfTraining = 0;
            dfTesting = 0;
 
            if (!m_colDatasets.ContainsKey(m_nStrIDHashCode))
                return 0;
 
            DatasetExCollection2 col = m_colDatasets[m_nStrIDHashCode];
            DatasetEx2 ds = col.FindDataset(strDataset);
 
            if (ds == null)
                return 0;
 
            return ds.GetPercentageLoaded(out dfTraining, out dfTesting);
        }
 
        public double GetDatasetLoadedPercentById(int nDatasetID, out double dfTraining, out double dfTesting)
        {
            dfTraining = 0;
            dfTesting = 0;
 
            if (!m_colDatasets.ContainsKey(m_nStrIDHashCode))
                return 0;
 
            DatasetExCollection2 col = m_colDatasets[m_nStrIDHashCode];
            DatasetEx2 ds = col.FindDataset(nDatasetID);
 
            if (ds == null)
                return 0;
 
            return ds.GetPercentageLoaded(out dfTraining, out dfTesting);
        }
 
        public DatasetDescriptor GetDatasetById(int nDsId)
        {
            DatasetEx2 ds = m_colDatasets[m_nStrIDHashCode].FindDataset(nDsId);
            return ds.Descriptor;
        }
 
        public DatasetDescriptor GetDatasetByName(string strDs)
        {
            DatasetEx2 ds = m_colDatasets[m_nStrIDHashCode].FindDataset(strDs);
            return ds.Descriptor;
        }
 
        public int GetDatasetID(string strDs)
        {
            DatasetDescriptor ds = GetDatasetByName(strDs);
            if (ds == null)
                return 0;
 
            return ds.ID;
        }
 
        public string GetDatasetName(int nDsId)
        {
            DatasetDescriptor ds = GetDatasetById(nDsId);
            if (ds == null)
                return null;
 
            return ds.Name;
        }
 
        public bool ReloadImageSet(int nSrcID)
        {
            ImageSet2 imgset = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcID);
            if (imgset != null)
            {
                imgset.ResetLabels();
                return true;
            }
 
            return false;
        }
 
        #endregion // Datasets
 
        #region Sources
 
        public SourceDescriptor GetSourceById(int nSrcId)
        {
            ImageSet2 imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId);
            if (imgSet != null)
                return imgSet.Source;
 
            return null;
        }
 
        public SourceDescriptor GetSourceByName(string strSrc)
        {
            ImageSet2 imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(strSrc);
            if (imgSet != null)
                return imgSet.Source;
 
            return null;
        }
 
        public int GetSourceID(string strSrc)
        {
            SourceDescriptor desc = GetSourceByName(strSrc);
            if (desc == null)
                return 0;
 
            return desc.ID;
        }
 
        public string GetSourceName(int nSrcId)
        {
            SourceDescriptor desc = GetSourceById(nSrcId);
            if (desc == null)
                return null;
 
            return desc.Name;
        }
 
        #endregion // Sources
 
        #region Labels
 
        public List<LabelDescriptor> GetLabels(int nSrcId)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetLabels();
        }
 
        public string GetLabelName(int nSrcId, int nLabel)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetLabelName(nLabel);
        }
 
        public void SetLabelMapping(int nSrcId, LabelMapping map)
        {
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).SetLabelMapping(map);
        }
 
        public void UpdateLabelMapping(int nSrcId, int nNewLabel, List<int> rgOriginalLabels)
        {
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).UpdateLabelMapping(nNewLabel, rgOriginalLabels);
        }
 
        public void ResetLabels(int nProjectId, int nSrcId)
        {
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).ResetLabels(nProjectId);
        }
 
        public void UpdateLabelCounts(int nProjectID, int nSrcId)
        {
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).UpdateLabelCounts(nProjectID);
        }
 
        public Dictionary<int, int> LoadLabelCounts(int nSrcId)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).LoadLabelCounts();
        }
 
        public string GetLabelCountsAsTextFromSourceId(int nSrcId)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetLabelCountsAsText();
        }
 
        public string GetLabelCountsAsTextFromSourceName(string strSource)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(strSource).GetLabelCountsAsText();
        }
 
        #endregion // Labels
 
        #region Boosts
 
        public void ResetAllBoosts(int nSrcId)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return;
 
            DatasetEx2 ds = m_colDatasets[m_nStrIDHashCode].FindDatasetFromSource(nSrcId);
            ds.ResetAllBoosts();
        }
 
        #endregion // Boosts
 
        #region Results
 
        public List<SimpleResult> GetAllResults(string strSource, bool bRequireExtraData, int nMax = -1)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(strSource).GetAllResults(bRequireExtraData, nMax);
        }
 
        #endregion // Results
    }
}