MyCaffeImageDatabase.cs

using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Drawing;
using System.Threading;
using MyCaffe.basecode;
using MyCaffe.basecode.descriptors;
 
namespace MyCaffe.db.image
{
    public partial class MyCaffeImageDatabase : Component, IXImageDatabase1
    {
        CryptoRandom m_random = null;
        DatasetFactory m_factory;
        string m_strID = "";
        int m_nStrIDHashCode = 0;
        EventWaitHandle m_evtInitializing = null;
        EventWaitHandle m_evtInitialized = null;
        EventWaitHandle m_evtAbortInitialization = null;
        bool m_bEnabled = false;
        static object m_syncObject = new object();
        static Dictionary<int, DatasetExCollection> m_colDatasets = new Dictionary<int, DatasetExCollection>();
        static Dictionary<int, LabelMappingCollection> m_rgLabelMappings = new Dictionary<int, LabelMappingCollection>();
        Dictionary<int, SimpleDatum> m_rgMeanCache = new Dictionary<int, SimpleDatum>();
        double m_dfSuperBoostProbability = 0;
        DB_ITEM_SELECTION_METHOD m_imageSelectionMethod = DB_ITEM_SELECTION_METHOD.RANDOM;
        DB_LABEL_SELECTION_METHOD m_labelSelectionMethod = DB_LABEL_SELECTION_METHOD.RANDOM;
        Log m_log;
        DB_LOAD_METHOD m_loadMethod = DB_LOAD_METHOD.LOAD_ON_DEMAND;
        int m_nLoadLimit = 0;
        bool m_bSkipMeanCheck = false;
        int m_nPadW = 0;
        int m_nPadH = 0;
        Guid m_userGuid;
 
 
        public event EventHandler<CalculateImageMeanArgs> OnCalculateImageMean;
 
        public MyCaffeImageDatabase(Log log = null, string strId = "default", int nSeed = 0)
        {
            m_factory = new DatasetFactory();
            m_userGuid = Guid.NewGuid();
            m_log = log;
            InitializeComponent();
            init(strId, nSeed);
 
            if (log != null)
                log.WriteLine("INFO: Using MyCaffe Image Database VERSION 1.");
        }
 
        public MyCaffeImageDatabase(IContainer container)
        {
            container.Add(this);
 
            InitializeComponent();
            init();
        }
 
        public DB_VERSION GetVersion()
        {
            return DB_VERSION.IMG_V1;
        }
 
        private void init(string strId = "", int nSeed = 0)
        {
            int nProcessID = Process.GetCurrentProcess().Id;
 
            m_random = new CryptoRandom(CryptoRandom.METHOD.DEFAULT, nSeed);
            m_strID = strId;
            m_nStrIDHashCode = strId.GetHashCode();
            m_evtInitializing = new EventWaitHandle(false, EventResetMode.ManualReset, "__CAFFE_IMAGEDATABASE__INITIALIZING__" + nProcessID.ToString());
            m_evtInitialized = new EventWaitHandle(false, EventResetMode.ManualReset, "__CAFFE_IMAGEDATABASE__INITIALIZED__" + nProcessID.ToString());
            m_evtAbortInitialization = new EventWaitHandle(false, EventResetMode.ManualReset, "__CAFFE_IMAGEDATABASE__ABORT_INITIALIZE__" + nProcessID.ToString());
        }
 
        private void dispose()
        {
            CleanUp();
 
            if (m_evtInitialized != null)
            {
                m_evtInitialized.Dispose();
                m_evtInitialized = null;
            }
 
            if (m_evtInitializing != null)
            {
                m_evtInitializing.Dispose();
                m_evtInitializing = null;
            }
 
            if (m_evtAbortInitialization != null)
            {
                m_evtAbortInitialization.Dispose();
                m_evtAbortInitialization = null;
            }
 
            if (m_random != null)
            {
                m_random.Dispose();
                m_random = null;
            }
        }
 
        public Log OutputLog
        {
            get { return m_log; }
            set { m_log = value; }
        }
 
        public void SetConnection(ConnectInfo ci)
        {
            MyCaffe.db.image.EntitiesConnection.GlobalDatabaseConnectInfo = ci;
        }
 
        public static Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD> GetSelectionMethod(SettingsCaffe s)
        {
            DB_ITEM_SELECTION_METHOD imageSelectionMethod = DB_ITEM_SELECTION_METHOD.NONE;
            DB_LABEL_SELECTION_METHOD labelSelectionMethod = DB_LABEL_SELECTION_METHOD.NONE;
 
            if (s.EnableRandomInputSelection)
                imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.RANDOM;
 
            if (s.SuperBoostProbability > 0)
                imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.BOOST;
 
            if (s.EnableLabelBalancing)
            {
                labelSelectionMethod |= DB_LABEL_SELECTION_METHOD.RANDOM;
 
                if (s.EnableLabelBoosting)
                    labelSelectionMethod |= DB_LABEL_SELECTION_METHOD.BOOST;
            }
            else
            {
                if (s.EnablePairInputSelection)
                    imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.PAIR;
            }
 
            return new Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD>(labelSelectionMethod, imageSelectionMethod);
        }
 
        public static Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD> GetSelectionMethod(ProjectEx p)
        {
            DB_ITEM_SELECTION_METHOD imageSelectionMethod = DB_ITEM_SELECTION_METHOD.NONE;
            DB_LABEL_SELECTION_METHOD labelSelectionMethod = DB_LABEL_SELECTION_METHOD.NONE;
 
            if (p.EnableRandomSelection)
                imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.RANDOM;
 
            if (p.EnableLabelBalancing)
            {
                labelSelectionMethod |= DB_LABEL_SELECTION_METHOD.RANDOM;
 
                if (p.EnableLabelBoosting)
                    labelSelectionMethod |= DB_LABEL_SELECTION_METHOD.BOOST;
            }
            else
            {
                if (p.EnablePairSelection)
                    imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.PAIR;
            }
 
            return new Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD>(labelSelectionMethod, imageSelectionMethod);
        }
 
        public Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD> GetSelectionMethod()
        {
            return new Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD>(m_labelSelectionMethod, m_imageSelectionMethod);
        }
 
        public void SetSelectionMethod(DB_LABEL_SELECTION_METHOD? lbl, DB_ITEM_SELECTION_METHOD? img)
        {
            if (lbl.HasValue)
                m_labelSelectionMethod = lbl.Value;
 
            if (img.HasValue)
                m_imageSelectionMethod = img.Value;
        }
 
        public bool GetLoadItemDataCriteria()
        {
            return m_factory.LoadDataCriteria;
        }
 
        public bool GetLoadItemDebugData()
        {
            return m_factory.LoadDebugData;
        }
 
        public bool UseTrainingImagesForTesting
        {
            get { return m_colDatasets[m_nStrIDHashCode].UseTrainingSourcesForTesting; }
            set { m_colDatasets[m_nStrIDHashCode].EnableUsingTrainingSourcesForTesting(value); }
        }
 
        public double SuperBoostProbability
        {
            get { return m_dfSuperBoostProbability; }
            set { m_dfSuperBoostProbability = value; }
        }
 
        public bool SelectFromBoostOnly
        {
            get
            {
                if (m_dfSuperBoostProbability > 0)
                {
                    double dfRandom = m_random.NextDouble(0, 1);
 
                    if (dfRandom <= m_dfSuperBoostProbability)
                        return true;
                }
 
                return false;
            }
        }
 
        public bool IsEnabled
        {
            get { return m_bEnabled; }
        }
 
        public void Enable(bool bEnable)
        {
            m_bEnabled = bEnable;
        }
 
        public bool IsInitialized
        {
            get { return m_evtInitialized.WaitOne(0); }
        }
 
        public bool InitializeWithDsName1(SettingsCaffe s, string strDs, string strEvtCancel = null, PropertySet prop = null)
        {
            return InitializeWithDs1(s, new DatasetDescriptor(strDs), strEvtCancel);
        }
 
        public bool InitializeWithDs1(SettingsCaffe s, DatasetDescriptor ds, string strEvtCancel = null, PropertySet prop = null)
        {
            string strDsName = ds.Name;
 
            if (String.IsNullOrEmpty(strDsName))
            {
                strDsName = m_factory.FindDatasetNameFromSourceName(ds.TrainingSourceName, ds.TestingSourceName);
                if (strDsName == null)
                    throw new Exception("Could not find the dataset! You must specify either the Datast name or at least the training or testing source name!");
            }
 
            int nDsId = m_factory.GetDatasetID(strDsName);
 
            return InitializeWithDsId1(s, nDsId, strEvtCancel);
        }
 
        public bool InitializeWithDsId1(SettingsCaffe s, int nDataSetID, string strEvtCancel = null, int nPadW = 0, int nPadH = 0, PropertySet prop = null)
        {
            Tuple<DB_LABEL_SELECTION_METHOD, DB_ITEM_SELECTION_METHOD> selMethod = GetSelectionMethod(s);
 
            m_nPadW = nPadW;
            m_nPadH = nPadH;
            m_labelSelectionMethod = selMethod.Item1;
            m_imageSelectionMethod = selMethod.Item2;
            m_dfSuperBoostProbability = s.SuperBoostProbability;
            m_loadMethod = s.DbLoadMethod;
            m_nLoadLimit = s.DbLoadLimit;
            m_bSkipMeanCheck = s.SkipMeanCheck;
 
            if (m_loadMethod == DB_LOAD_METHOD.LOAD_EXTERNAL)
                m_loadMethod = DB_LOAD_METHOD.LOAD_ON_DEMAND;
 
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtInitialized, m_evtInitializing, m_evtAbortInitialization }, 0);
 
            lock (m_syncObject)
            {
                if (nWait != WaitHandle.WaitTimeout)
                {
                    if (nWait == 0)     // already initialized.
                    {
                        if (m_colDatasets.ContainsKey(m_nStrIDHashCode))
                        {
                            DatasetExCollection col = m_colDatasets[m_nStrIDHashCode];
 
                            if (m_log != null)
                                m_log.WriteLine("The MyCaffe Image Database is already initialized.");
 
                            DatasetEx ds = col.FindDataset(nDataSetID);
                            if (ds != null)
                            {
                                ds.AddUser(m_userGuid);
                                return true;
                            }
                        }
                    }
                    else
                    {
                        return false;
                    }
                }
 
                try
                {
                    m_evtInitializing.Set();
 
                    m_factory.SetLoadingParameters(s.ItemDbLoadDataCriteria, s.ItemDbLoadDebugData);
 
                    DatasetDescriptor ds = m_factory.LoadDataset(nDataSetID);
                    if (ds == null)
                        throw new Exception("Could not find dataset with ID = " + nDataSetID.ToString());
 
                    List<WaitHandle> rgAbort = new List<WaitHandle>() { m_evtAbortInitialization };
                    CancelEvent evtCancel;
 
                    if (strEvtCancel != null)
                    {
                        evtCancel = new CancelEvent(strEvtCancel);
                        rgAbort.AddRange(evtCancel.Handles);
                    }
 
                    DatasetExCollection col = null;
 
                    if (m_colDatasets.ContainsKey(m_nStrIDHashCode))
                        col = m_colDatasets[m_nStrIDHashCode];
                    else
                        col = new DatasetExCollection();
 
                    if (m_evtAbortInitialization.WaitOne(0))
                    {
                        col.Dispose();
                        return false;
                    }
 
                    DatasetEx ds0 = new DatasetEx(m_userGuid, m_factory, m_random);
 
                    if (OnCalculateImageMean != null)
                        ds0.OnCalculateImageMean += OnCalculateImageMean;
 
                    if (m_log != null)
                        m_log.WriteLine("Loading dataset '" + ds.Name + "'...");
 
                    if (!ds0.Initialize(ds, rgAbort.ToArray(), nPadW, nPadH, m_log, m_loadMethod, m_nLoadLimit, m_bSkipMeanCheck))
                    {
                        col.Dispose();
                        return false;
                    }
 
                    if (m_log != null)
                        m_log.WriteLine("Dataset '" + ds.Name + "' loaded.");
 
                    col.Add(ds0);
 
                    if (!m_colDatasets.ContainsKey(m_nStrIDHashCode))
                        m_colDatasets.Add(m_nStrIDHashCode, col);
 
                    UseTrainingImagesForTesting = s.UseTrainingSourceForTesting;
 
                    return true;
                }
                finally
                {
                    m_evtInitialized.Set();
                    m_evtInitializing.Reset();
                }
            }
        }
 
        public void CleanUp(int nDsId = 0, bool bForce = false)
        {
            lock (m_syncObject)
            {
                if (m_evtInitializing.WaitOne(0))
                {
                    m_evtAbortInitialization.Set();
                    return;
                }
 
                List<int> rgRemove = new List<int>();
 
                foreach (KeyValuePair<int, DatasetExCollection> col in m_colDatasets)
                {
                    DatasetExCollection colDs = col.Value;
 
                    if (colDs.RemoveUser(m_userGuid) || bForce)
                    {
                        rgRemove.Add(col.Key);
                        colDs.Dispose();
                    }
                }
 
                foreach (int nKey in rgRemove)
                {
                    m_colDatasets.Remove(nKey);
                }
 
                if (m_colDatasets.Count == 0)
                {
                    if (m_evtInitialized.WaitOne(0))
                        m_evtInitialized.Reset();
                }
            }
        }
 
        public bool LoadNextSet(string strEvtCancel)
        {
            if (m_nLoadLimit == 0)
                return false;
 
            lock (m_syncObject)
            {
                if (!m_evtInitialized.WaitOne(0))
                    return false;
 
                List<WaitHandle> rgAbort = new List<WaitHandle>() { m_evtAbortInitialization };
                CancelEvent evtCancel;
 
                if (strEvtCancel != null)
                {
                    evtCancel = new CancelEvent(strEvtCancel);
                    rgAbort.AddRange(evtCancel.Handles);
                }
 
                if (!m_colDatasets.ContainsKey(m_nStrIDHashCode))
                    return false;
 
                DatasetExCollection col = m_colDatasets[m_nStrIDHashCode];
                col.Reset();
 
                foreach (DatasetEx ds in col)
                {
                    if (m_evtAbortInitialization.WaitOne(0))
                        return false;
 
                    if (m_log != null)
                        m_log.WriteLine("Loading next limited image set for dataset '" + ds.DatasetName + "'...");
 
                    if (!ds.Initialize(null, rgAbort.ToArray(), m_nPadW, m_nPadH, m_log, m_loadMethod, m_nLoadLimit))
                        return false;
 
                    if (m_log != null)
                        m_log.WriteLine("Dataset '" + ds.DatasetName + "' re-loaded.");
                }
 
                return true;
            }
        }
 
        public void UpdateLabelBoosts(int nProjectID, int nSrcID)
        {
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcID).UpdateLabelBoosts(nProjectID);
        }
 
        public int ImageCount(int nSrcId)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
 
            if (nWait == 0)
                return 0;
 
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).Count;
        }
 
        public int GetItemCount(int nSrcId, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
 
            if (nWait == 0)
                return 0;
 
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetCount(nBoostVal.HasValue, strFilterVal, nBoostVal);
        }
 
        public List<SimpleDatum> GetItemsFromIndex(int nSrcId, int nStartIdx, int nQueryCount = int.MaxValue, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false, bool bAttemptDirectLoad = false)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return null;
 
            if (bAttemptDirectLoad)
                throw new Exception("Direct load attempts are not supported by the Image Database V1.");
 
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetImages(nBoostVal.HasValue, strFilterVal, nBoostVal, nStartIdx, nQueryCount);
        }
 
        public List<SimpleDatum> GetItemsFromTime(int nSrcId, DateTime dtStart, int nQueryCount = int.MaxValue, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return null;
 
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetImages(dtStart, nQueryCount, strFilterVal);
        }
 
        public List<SimpleDatum> GetItems(int nSrcId, int[] rgIdx, string strFilterVal = null, int? nBoostVal = null, bool bBoostValIsExact = false)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return null;
 
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetImages(nBoostVal.HasValue, strFilterVal, nBoostVal, rgIdx);
        }
 
        public bool Sort(int nSrcId, IMGDB_SORT method)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
 
            if (nWait == 0)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).Sort(method);
        }
 
        public int CreateDatasetOranizedByTime(int nDsId)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
 
            if (nWait == 0)
                return 0;
 
            // If we have already created a re-organized dataset,
            //  return its DatasetID.
            foreach (DatasetEx ds1 in m_colDatasets[m_nStrIDHashCode])
            {
                if (ds1.OriginalDatasetID == nDsId)
                    return ds1.DatasetID;
            }
 
            DatasetEx ds = m_colDatasets[m_nStrIDHashCode].FindDataset(nDsId);
            DatasetEx dsNew = ds.Clone(true);
 
            m_colDatasets[m_nStrIDHashCode].Add(dsNew);
 
            return dsNew.Descriptor.ID;
        }
 
        public bool DeleteCreatedDataset(int nDsId)
        {
            if (nDsId >= 0)
                throw new Exception("The dataset specified is not a dynamic dataset.");
 
            DatasetEx ds = m_colDatasets[m_nStrIDHashCode].FindDataset(nDsId);
            if (ds == null)
                return false;
 
            return m_colDatasets[m_nStrIDHashCode].RemoveDataset(ds);
        }
 
        public void DeleteAllCreatedDatasets()
        {
            m_colDatasets[m_nStrIDHashCode].RemoveCreatedDatasets();
        }
 
        public SimpleDatum QueryItem(int nSrcId, int nIdx, DB_LABEL_SELECTION_METHOD? labelSelectionOverride = null, DB_ITEM_SELECTION_METHOD? imageSelectionOverride = null, int? nLabel = null, bool bLoadDataCriteria = false, bool bLoadDebugData = false)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
 
            if (nWait == 0)
                return null;
 
            DB_LABEL_SELECTION_METHOD labelSelectionMethod = m_labelSelectionMethod;
            DB_ITEM_SELECTION_METHOD imageSelectionMethod = m_imageSelectionMethod;
 
            if (labelSelectionOverride.HasValue)
                labelSelectionMethod = labelSelectionOverride.Value;
 
            if (imageSelectionOverride.HasValue)
                imageSelectionMethod = imageSelectionOverride.Value;
            else if (SelectFromBoostOnly)
                imageSelectionMethod |= DB_ITEM_SELECTION_METHOD.BOOST;
 
            SimpleDatum sd = null;
            ImageSet imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId);
            LabelSet lblSet = null;
 
            if (nLabel.HasValue)
            {
                lblSet = imgSet.GetLabelSet(nLabel.Value);
                if (lblSet.IsLoaded)
                    sd = lblSet.GetImage(0, imageSelectionMethod);
            }
           
            if ((m_loadMethod == DB_LOAD_METHOD.LOAD_ON_DEMAND || m_loadMethod == DB_LOAD_METHOD.LOAD_ON_DEMAND_NOCACHE) && (imageSelectionMethod & DB_ITEM_SELECTION_METHOD.PAIR) == DB_ITEM_SELECTION_METHOD.PAIR)
                throw new Exception("PAIR selection is not supported whith the LOAD_ON_DEMAND loading method.");
 
            if (sd == null)
            {
                sd = imgSet.GetImage(nIdx, labelSelectionMethod, imageSelectionMethod, m_log, bLoadDataCriteria, bLoadDebugData);
                if (sd == null)
                {
                    Exception err = new Exception("Could not acquire an image - re-index the dataset.");
                    m_log.WriteError(err);
                    throw err;
                }
 
                if (nLabel.HasValue)
                {
                    while (sd.Label != nLabel.Value)
                    {
                        sd = imgSet.GetImage(nIdx, labelSelectionMethod, DB_ITEM_SELECTION_METHOD.RANDOM, m_log, bLoadDataCriteria, bLoadDebugData);
                    }
 
                    lblSet.Add(sd);
                }
            }
 
            imgSet.SetQueryLabelCount(sd.Label, sd.Boost);
 
            return sd;
        }
 
        public void ResetQuery(int nDsID)
        {
            DatasetDescriptor ds = GetDatasetById(nDsID);
 
            ImageSet imgSetTrain = m_colDatasets[m_nStrIDHashCode].FindImageset(ds.TrainingSource.ID);
            imgSetTrain.Reset();
 
            ImageSet imgSetTest = m_colDatasets[m_nStrIDHashCode].FindImageset(ds.TestingSource.ID);
            imgSetTest.Reset();
        }
 
        public SimpleDatum GetItem(int nImageID, params int[] rgSrcId)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
 
            if (nWait == 0)
                return null;
 
            foreach (int nSrcId in rgSrcId)
            {
                ImageSet imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId);
                SimpleDatum sd = imgSet.GetImage(nImageID);
 
                if (sd != null)
                    return sd;
            }
 
            return null;
        }
 
        public void ResetAllBoosts(int nSrcId)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
            if (nWait == 0)
                return;
 
            ImageSet imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId);
            imgSet.ResetAllBoosts();
        }
 
        public List<LabelDescriptor> GetLabels(int nSrcId)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetLabels();
        }
 
        public string GetLabelName(int nSrcId, int nLabel)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetLabelName(nLabel);
        }
 
        public DatasetDescriptor GetDatasetById(int nDsId)
        {
            DatasetEx ds = m_colDatasets[m_nStrIDHashCode].FindDataset(nDsId);
            return ds.Descriptor;
        }
 
        public DatasetDescriptor GetDatasetByName(string strDs)
        {
            DatasetEx ds = m_colDatasets[m_nStrIDHashCode].FindDataset(strDs);
            return ds.Descriptor;
        }
 
        public int GetDatasetID(string strDs)
        {
            DatasetDescriptor ds = GetDatasetByName(strDs);
            if (ds == null)
                return 0;
 
            return ds.ID;
        }
 
        public string GetDatasetName(int nDsId)
        {
            DatasetDescriptor ds = GetDatasetById(nDsId);
            if (ds == null)
                return null;
 
            return ds.Name;
        }
 
        public SourceDescriptor GetSourceById(int nSrcId)
        {
            ImageSet imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId);
            if (imgSet != null)
                return imgSet.Source;
 
            return null;
        }
 
        public SourceDescriptor GetSourceByName(string strSrc)
        {
            ImageSet imgSet = m_colDatasets[m_nStrIDHashCode].FindImageset(strSrc);
            if (imgSet != null)
                return imgSet.Source;
 
            return null;
        }
 
        public int GetSourceID(string strSrc)
        {
            SourceDescriptor desc = GetSourceByName(strSrc);
            if (desc == null)
                return 0;
 
            return desc.ID;
        }
 
        public string GetSourceName(int nSrcId)
        {
            SourceDescriptor desc = GetSourceById(nSrcId);
            if (desc == null)
                return null;
 
            return desc.Name;
        }
 
        public int FindItemIndex(int nSrcId, DateTime dt, string strDescription)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).FindImageIndex(dt, strDescription);
        }
 
        public SimpleDatum GetItemMean(int nSrcId)
        {
            if (m_evtAbortInitialization.WaitOne(0))
                return null;
 
            if (!m_evtInitialized.WaitOne(0))
            {
                if (m_rgMeanCache.Keys.Contains(nSrcId))
                    return m_rgMeanCache[nSrcId];
            }
 
            SimpleDatum sd = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetImageMean(null, null);
 
            if (!m_rgMeanCache.ContainsKey(nSrcId))
                m_rgMeanCache.Add(nSrcId, sd);
            else
                m_rgMeanCache[nSrcId] = sd;
 
            return sd;
        }
 
        public void SetImageMean(int nSrcId, SimpleDatum d)
        {
            int nWait = WaitHandle.WaitAny(new WaitHandle[] { m_evtAbortInitialization, m_evtInitialized });
 
            if (nWait == 0)
                return;
 
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).SetImageMean(d);
        }
 
        public SimpleDatum QueryItemMeanFromDataset(int nDatasetId)
        {
            DatasetEx ds = m_colDatasets[m_nStrIDHashCode].FindDataset(nDatasetId);
            if (ds == null)
                return null;
 
            return QueryItemMean(ds.Descriptor.TrainingSource.ID);
        }
 
        public SimpleDatum QueryItemMeanFromDb(int nSrcId)
        {
            SimpleDatum sd = QueryItemMean(nSrcId);
 
            if (sd != null)
                return sd;
 
            sd = GetItemMean(nSrcId);
            SaveImageMean(nSrcId, sd, false);
 
            return sd;
        }
 
        public void SaveImageMean(int nSrcId, SimpleDatum d, bool bUpdate)
        {
            if (m_colDatasets.ContainsKey(m_nStrIDHashCode))
                m_colDatasets[m_nStrIDHashCode].SaveImageMean(nSrcId, d, bUpdate);
        }
 
        public SimpleDatum QueryItemMean(int nSrcId)
        {
            if (!m_colDatasets.ContainsKey(m_nStrIDHashCode))
                return null;
 
            return m_colDatasets[m_nStrIDHashCode].QueryImageMean(nSrcId);
        }
 
        public bool DoesImageMeanExists(int nSrcId)
        {
            using (DNNEntities entities = EntitiesConnection.CreateEntities())
            {
                List<Source> rgSrc = entities.Sources.Where(p => p.ID == nSrcId).ToList();
                if (rgSrc.Count == 0)
                    return false;
 
                IQueryable<RawImageMean> iQuery = entities.RawImageMeans.Where(p => p.SourceID == nSrcId);
                if (iQuery != null)
                {
                    List<RawImageMean> rgMean = iQuery.ToList();
                    if (rgMean.Count > 0)
                        return true;
                }
 
                return false;
            }
        }
 
        public void SetLabelMapping(int nSrcId, LabelMapping map)
        {
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).SetLabelMapping(map);
        }
 
        public void UpdateLabelMapping(int nSrcId, int nNewLabel, List<int> rgOriginalLabels)
        {
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).UpdateLabelMapping(nNewLabel, rgOriginalLabels);
        }
 
        public void ResetLabels(int nProjectId, int nSrcId)
        {
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).ResetLabels(nProjectId);
        }
 
        public void DeleteLabelBoosts(int nProjectId, int nSrcId)
        {
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).DeleteLabelBoosts(nProjectId);
        }
 
        public void AddLabelBoost(int nProjectID, int nSrcId, int nLabel, double dfBoost)
        {
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).AddLabelBoost(nProjectID, nLabel, dfBoost);
        }
 
        public string GetLabelBoostsAsTextFromProject(int nProjectId, int nSrcId)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetLabelBoostsAsText(nProjectId);
        }
 
        public void UpdateLabelCounts(int nProjectID, int nSrcId)
        {
            m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).UpdateLabelCounts(nProjectID);
        }
 
        public Dictionary<int, int> LoadLabelCounts(int nSrcId)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).LoadLabelCounts();
        }
 
        public string GetLabelCountsAsTextFromSourceId(int nSrcId)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcId).GetLabelCountsAsText();
        }
 
        public string GetLabelCountsAsTextFromSourceName(string strSource)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(strSource).GetLabelCountsAsText();
        }
 
        public string GetLabelQueryHitPercentsAsTextFromSourceName(string strSource)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(strSource).GetQueryLabelHitPrecentsAsText();
        }
 
        public string GetLabelQueryEpocsAsTextFromSourceName(string strSource)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(strSource).GetQueryLabelEpocsAsText();
        }
 
        public string GetBoostQueryHitPercentsAsTextFromSourceName(string strSource)
        {
            return m_colDatasets[m_nStrIDHashCode].FindImageset(strSource).GetQueryBoostHitPrecentsAsText();
        }
 
        public bool LoadDatasetByName1(string strDs, string strEvtCancel = null)
        {
            int nDsId = m_factory.GetDatasetID(strDs);
            return LoadDatasetByID1(nDsId, strEvtCancel);
        }
 
 
        public bool LoadDatasetByID1(int nDsId, string strEvtCancel = null)
        {
            if (!m_evtInitialized.WaitOne(0))
                throw new Exception("The image database must be initialized first before a secondary dataset can be loaded.");
 
            if (m_evtInitializing.WaitOne(0))
                throw new Exception("The image database is in the process of being initialized.");
 
            DatasetEx ds = m_colDatasets[m_nStrIDHashCode].FindDataset(nDsId);
            if (ds != null)
                return false;
 
            DatasetDescriptor desc = m_factory.LoadDataset(nDsId);
            if (desc == null)
                throw new Exception("Could not find dataset with ID = " + nDsId.ToString());
 
            if (!m_colDatasets.ContainsKey(m_nStrIDHashCode))
                throw new Exception("The image database was not initialized properly.");
 
            DatasetExCollection col = m_colDatasets[m_nStrIDHashCode];
            DatasetEx ds0 = new DatasetEx(m_userGuid, m_factory, m_random);
 
            if (OnCalculateImageMean != null)
                ds0.OnCalculateImageMean += OnCalculateImageMean;
 
            if (m_log != null)
                m_log.WriteLine("Loading dataset '" + desc.Name + "'...");
 
            CancelEvent evtCancel = new CancelEvent(strEvtCancel);
            List<WaitHandle> rgAbort = new List<WaitHandle>(evtCancel.Handles);
 
            if (!ds0.Initialize(desc, rgAbort.ToArray(), 0, 0, m_log, m_loadMethod, m_nLoadLimit))
            {
                col.Dispose();
                return false;
            }
 
            if (m_log != null)
                m_log.WriteLine("Dataset '" + desc.Name + "' loaded.");
 
            col.Add(ds0);
 
            return true;
        }
 
        public bool ReloadDataset(int nDsId)
        {
            DatasetEx ds = m_colDatasets[m_nStrIDHashCode].FindDataset(nDsId);
            if (ds != null)
            {
                ds.ReloadLabelSets();
                return true;
            }
 
            return false;
        }
 
        public bool ReloadImageSet(int nSrcID)
        {
            ImageSet imgset = m_colDatasets[m_nStrIDHashCode].FindImageset(nSrcID);
            if (imgset != null)
            {
                imgset.ReloadLabelSets();
                return true;
            }
 
            return false;
        }
 
        public bool UnloadDatasetByName(string strDataset)
        {
            bool bRemoved = false;
 
            lock (m_syncObject)
            {
                if (m_colDatasets.ContainsKey(m_nStrIDHashCode))
                {
                    DatasetExCollection col = m_colDatasets[m_nStrIDHashCode];
                    DatasetEx ds = col.FindDataset(strDataset);
                    if (ds != null)
                    {
                        if (m_log != null)
                            m_log.WriteLine("Unloading dataset '" + ds.DatasetName + "'.");
 
                        ds.Unload();
                        GC.Collect();
                        bRemoved = true;
                    }
                }
            }
 
            return bRemoved;
        }
 
        public bool UnloadDatasetById(int nDataSetID)
        {
            bool bRemoved = false;
 
            lock (m_syncObject)
            {
                if (m_colDatasets.ContainsKey(m_nStrIDHashCode))
                {
                    DatasetExCollection col = m_colDatasets[m_nStrIDHashCode];
 
                    foreach (DatasetEx ds in col)
                    {
                        if (ds != null)
                        {
                            if (ds.DatasetID == nDataSetID || nDataSetID == -1)
                            {
                                if (m_log != null)
                                    m_log.WriteLine("Unloading dataset '" + ds.DatasetName + "'.");
 
                                ds.Unload();
                                GC.Collect();
                                bRemoved = true;
                            }
                        }
                    }
                }
            }
 
            return bRemoved;
        }
 
 
        public double GetDatasetLoadedPercentByName(string strDataset, out double dfTraining, out double dfTesting)
        {
            dfTraining = 0;
            dfTesting = 0;
 
            if (!m_colDatasets.ContainsKey(m_nStrIDHashCode))
                return 0;
 
            DatasetExCollection col = m_colDatasets[m_nStrIDHashCode];
            DatasetEx ds = col.FindDataset(strDataset);
 
            if (ds == null)
                return 0;
 
            return ds.GetPercentageLoaded(out dfTraining, out dfTesting);
        }
 
        public double GetDatasetLoadedPercentById(int nDatasetID, out double dfTraining, out double dfTesting)
        {
            dfTraining = 0;
            dfTesting = 0;
 
            if (!m_colDatasets.ContainsKey(m_nStrIDHashCode))
                return 0;
 
            DatasetExCollection col = m_colDatasets[m_nStrIDHashCode];
            DatasetEx ds = col.FindDataset(nDatasetID);
 
            if (ds == null)
                return 0;
 
            return ds.GetPercentageLoaded(out dfTraining, out dfTesting);
        }
 
        public static void CreateDatabase(string strName, string strPath)
        {
            EntitiesConnection.GlobalDatabaseConnectInfo.Database = strName;
 
            DatabaseManagement dbMgr = new DatabaseManagement(EntitiesConnection.GlobalDatabaseConnectInfo, strPath);
            Exception excpt = dbMgr.CreateDatabase();
 
            if (excpt != null)
                throw excpt;
        }
    }
}