DatasetFactory.cs

using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using MyCaffe.basecode;
using MyCaffe.basecode.descriptors;
 
namespace MyCaffe.db.image
{
    public class DatasetFactory : IDisposable
    {
        protected Database m_db = new Database();
        protected SourceDescriptor m_openSource = null;
        protected int? m_nOriginalSourceID = null;
        protected bool m_bLoadDataCriteria = false;
        protected bool m_bLoadDebugData = false;
        protected ConnectInfo m_ciOpen = null;
 
        ImageCache m_imageCache = null;
        ParamCache m_paramCache = null;
 
 
        public DatasetFactory()
        {
            m_db = new Database();
        }
 
        public DatasetFactory(bool bLoadDataCriteria, bool bLoadDebugData)
        {
            m_bLoadDataCriteria = bLoadDataCriteria;
            m_bLoadDebugData = bLoadDebugData;
            m_db = new Database();
        }
 
        public DatasetFactory(DatasetFactory factory)
        {
            m_bLoadDebugData = factory.m_bLoadDebugData;
            m_bLoadDataCriteria = factory.m_bLoadDataCriteria;
        }
 
        public DatasetFactory(int nSrcId)
        {
            Open(nSrcId);
        }
 
        public void Dispose()
        {
        }
 
        public Database Database
        {
            get { return m_db; }
        }
 
        public int LastIndex
        {
            get { return m_db.LastIndex; }
        }
 
        public void SetLoadingParameters(bool bLoadDataCriteria, bool bLoadDebugData)
        {
            m_bLoadDataCriteria = bLoadDataCriteria;
            m_bLoadDebugData = bLoadDebugData;
        }
 
        public bool LoadDataCriteria
        {
            get { return m_bLoadDataCriteria; }
        }
 
        public bool LoadDebugData
        {
            get { return m_bLoadDebugData; }
        }
 
        public bool ChangeRawImageSourceID(int nID, int nNewSrcID, bool bSave = true)
        {
            return m_db.ChangeRawImageSourceID(nID, nNewSrcID, bSave);
        }
 
        public void SaveChanges()
        {
            m_db.SaveChanges();
        }
 
        public void Open(SourceDescriptor src, int nCacheMax = 500, ConnectInfo ci = null)
        {
            m_openSource = src;
            Open(src.ID, nCacheMax, Database.FORCE_LOAD.NONE, null, ci);
        }
 
        public void Open(int nSrcId, int nCacheMax = 500, Database.FORCE_LOAD nForceLoad = Database.FORCE_LOAD.NONE, Log log = null, ConnectInfo ci = null)
        {
            if (m_openSource != null)
            {
                if (m_openSource.ID != nSrcId)
                {
                    if (log != null)
                        log.WriteLine("WARNING: Closing currently open source ID of " + m_openSource.ID.ToString() + " and opening new Source ID of " + nSrcId.ToString() + ".");
                    Close();
                }
            }
 
            m_ciOpen = ci;
            m_openSource = LoadSource(nSrcId, ci);
            m_db.Open(nSrcId, nForceLoad, ci);
 
            m_imageCache = new ImageCache(nCacheMax);
            m_paramCache = new ParamCache(nCacheMax);
        }
 
        public void Close()
        {
            m_openSource = null;
            m_ciOpen = null;
            m_db.Close();
        }
 
        public void Refresh()
        {
            m_db.Refresh();
        }
 
        public SourceDescriptor OpenSource
        {
            get { return m_openSource; }
        }
 
 
        //---------------------------------------------------------------------
        //  Raw Images
        //---------------------------------------------------------------------
        #region RawImages
 
        public void PutRawImageParameterCache(int nImageID, string strParam, string strVal, double? dfVal, byte[] rgData, bool bOnlyAddNew)
        {
            if (m_paramCache.Add(new ParameterData(strParam, strVal, dfVal, rgData, nImageID, bOnlyAddNew, m_openSource.ID)))
                ClearParamCache(true);
        }
 
        public void ClearParamCache(bool bSave)
        {
            if (m_paramCache.Count == 0)
                return;
 
            if (bSave)
                m_db.PutRawImageParameters(m_paramCache.Parameters);
 
            m_paramCache.Clear();
        }
 
        public void PutRawImageCache(int nIdx, SimpleDatum sd, int nBackgroundWritingThreadCount = 0, string strDescription = null, bool bActive = true, params ParameterData[] rgParams)
        {
            RawImage img = m_db.CreateRawImage(nIdx, sd, nBackgroundWritingThreadCount, strDescription, m_nOriginalSourceID, bActive);
            if (m_imageCache.Add(img, sd, rgParams))
                ClearImageCache(true);
        }
 
        public void ClearImageCache(bool bSave)
        {
            if (m_imageCache == null || m_imageCache.Count == 0)
                return;
 
            if (bSave)
            {
                m_db.WaitForFileWriter();
                m_db.PutRawImages(m_imageCache.Images, m_imageCache.Parameters, m_ciOpen);
            }
 
            m_imageCache.Clear();
        }
 
        public int PutRawImage(int nIdx, SimpleDatum sd, string strDescription = null)
        {
            return m_db.PutRawImage(nIdx, sd, strDescription);
        }
 
        public int SetRawImageParameter(int nRawImageID, string strName, string strValue, double? dfVal = null, byte[] rgData = null, bool bOnlyAddNew = false)
        {
            return m_db.SetRawImageParameter(nRawImageID, strName, strValue, dfVal, rgData, true, bOnlyAddNew);
        }
 
        public int SetRawImageParameter(int nSrcID, int nRawImageID, string strName, string strValue, double? dfVal = null, byte[] rgData = null)
        {
            return m_db.SetRawImageParameter(nSrcID, nRawImageID, strName, strValue, dfVal, rgData);
        }
 
        public int SetRawImageParameterAt(DateTime dt, string strName, string strValue, double? dfVal = null, byte[] rgData = null)
        {
            return m_db.SetRawImageParameterAt(dt, strName, strValue, dfVal, rgData);
        }
 
        public List<RawImageParameter> QueryRawImageParameters(int nSrcId, string strName)
        {
            return m_db.QueryRawImageParameters(nSrcId, strName);
        }
 
        public List<RawImageParameter> QueryRawImageParameters(int nImageID)
        {
            return m_db.QueryRawImageParameters(nImageID);
        }
 
        public RawImageMean GetRawImageMean()
        {
            return m_db.GetRawImageMean();
        }
 
        public List<RawImage> GetRawImagesAt(int nImageIdx, int nImageCount, int nSrcId = 0, string strDescription = null)
        {
            return m_db.GetRawImagesAt(nImageIdx, nImageCount, nSrcId, strDescription);
        }
 
        public List<RawImage> GetRawImagesAt(List<int> rgImageIdx, ManualResetEvent evtCancel, int nSrcId = 0, string strDescription = null)
        {
            List<RawImage> rgImg = new List<RawImage>();
 
            while (rgImageIdx.Count > 0)
            {
                List<int> rgIdx = new List<int>();
 
                for (int i=0; i<rgImageIdx.Count && i < 100; i++)
                {
                    rgIdx.Add(rgImageIdx[i]);
                }
 
                List<RawImage> rgImg1 = m_db.GetRawImagesAt(rgIdx, nSrcId, strDescription);
                rgImg.AddRange(rgImg1);
 
                for (int i = 0; i < rgIdx.Count; i++)
                {
                    rgImageIdx.RemoveAt(0);
                }
 
                if (evtCancel.WaitOne(0))
                    return null;
            }
 
            return rgImg;
        }
 
        public List<RawImage> GetRawImagesAtID(List<int> rgImageId, ManualResetEvent evtCancel, int nSrcId = 0, string strDescription = null)
        {
            List<RawImage> rgImg = new List<RawImage>();
 
            while (rgImageId.Count > 0)
            {
                List<int> rgIdx = new List<int>();
 
                for (int i = 0; i < rgImageId.Count && i < 100; i++)
                {
                    rgIdx.Add(rgImageId[i]);
                }
 
                List<RawImage> rgImg1 = m_db.GetRawImagesAtID(rgIdx, nSrcId, strDescription);
                rgImg.AddRange(rgImg1);
 
                for (int i = 0; i < rgIdx.Count; i++)
                {
                    rgImageId.RemoveAt(0);
                }
 
                if (evtCancel.WaitOne(0))
                    return null;
            }
 
            return rgImg;
        }
 
        public List<RawImage> GetRawImagesAt(List<DbItem> rgImgItems, ManualResetEvent evtCancel, int nSrcId = 0, string strDescription = null)
        {
            List<RawImage> rgImg = new List<RawImage>();
 
            while (rgImgItems.Count > 0)
            {
                List<int> rgID = new List<int>();
 
                for (int i = 0; i < rgImgItems.Count && i < 100; i++)
                {
                    rgID.Add(rgImgItems[i].ID);
                }
 
                List<RawImage> rgImg1 = m_db.GetRawImagesAtID(rgID, nSrcId, strDescription);
                rgImg.AddRange(rgImg1);
 
                for (int i = 0; i < rgID.Count; i++)
                {
                    rgImgItems.RemoveAt(0);
                }
 
                if (evtCancel.WaitOne(0))
                    return null;
            }
 
            return rgImg;
        }
 
        public List<SimpleDatum> GetImagesAt(List<int> rgImageIdx, ManualResetEvent evtCancel, int nSrcId = 0, string strDescription = null)
        {
            List<RawImage> rgImg = GetRawImagesAt(rgImageIdx, evtCancel, nSrcId, strDescription);
            if (rgImg == null)
                return null;
 
            List<SimpleDatum> rgSd = new List<SimpleDatum>();
 
            foreach (RawImage img in rgImg)
            {
                rgSd.Add(LoadDatum(img));
            }
 
            return rgSd;
        }
 
        public List<SimpleDatum> GetImagesAt(List<DbItem> rgImageItems, ManualResetEvent evtCancel, int nSrcId = 0, string strDescription = null)
        {
            List<RawImage> rgImg = GetRawImagesAt(rgImageItems, evtCancel, nSrcId, strDescription);
            if (rgImg == null)
                return null;
 
            List<SimpleDatum> rgSd = new List<SimpleDatum>();
 
            foreach (RawImage img in rgImg)
            {
                rgSd.Add(LoadDatum(img));
            }
 
            return rgSd;
        }
 
        public List<SimpleDatum> GetImagesAtID(List<int> rgImageId, int nSrcId = 0, string strDescription = null)
        {
            List<RawImage> rgImg = m_db.GetRawImagesAtID(rgImageId, nSrcId, strDescription);
            if (rgImg == null)
                return null;
 
            List<SimpleDatum> rgSd = new List<SimpleDatum>();
 
            foreach (RawImage img in rgImg)
            {
                rgSd.Add(LoadDatum(img));
            }
 
            return rgSd;
        }
 
        public List<RawImage> QueryRawImages(int nSrcId, bool? bActive = null, bool bLoadCriteria = false, bool bLoadDebug = false, Log log = null, CancelEvent evtCancel = null, int nBoostVal = 0, bool bExactBoostVal = false)
        {
            List<RawImage> rgImg = m_db.QueryRawImages(nSrcId, bActive, nBoostVal, bExactBoostVal);
 
            if (!bLoadCriteria && !bLoadDebug)
                return rgImg;
 
            Stopwatch sw = new Stopwatch();
            sw.Start();
 
            for (int i = 0; i < rgImg.Count; i++)
            {
                int? nFmt;
 
                if (bLoadCriteria)
                    rgImg[i].DataCriteria = m_db.GetRawImageDataCriteria(rgImg[i], out nFmt);
 
                if (bLoadDebug)
                    rgImg[i].DebugData = m_db.GetRawImageDebugData(rgImg[i], out nFmt);
 
                if (evtCancel != null && evtCancel.WaitOne(0))
                    return null;
 
                if (log != null)
                {
                    if (sw.Elapsed.TotalMilliseconds > 1000)
                    {
                        log.Progress = (double)i / (double)rgImg.Count;
                        log.WriteLine("loading " + i.ToString("N0") + " of " + rgImg.Count.ToString("N0") + "...");
                        sw.Restart();
                    }
                }
            }
 
            return rgImg;
        }
 
        public void LoadRawImageData(RawImage img, bool bLoadCriteria, bool bLoadDebug)
        {
            int? nFmt;
 
            if (bLoadCriteria)
                img.DataCriteria = m_db.GetRawImageDataCriteria(img, out nFmt);
 
            if (bLoadDebug)
                img.DebugData = m_db.GetRawImageDebugData(img, out nFmt);
        }
 
        public int PutRawImageMean(SimpleDatum sd, bool bUpdate, ConnectInfo ci = null)
        {
            return m_db.PutRawImageMean(sd, bUpdate, 0, ci);
        }
 
        public bool SaveImageMean(SimpleDatum sd, bool bUpdate, int nSrcId = 0)
        {
            if (m_db.PutRawImageMean(sd, bUpdate, nSrcId) != 0)
                return true;
 
            return false;
        }
 
        public SimpleDatum QueryImageMean(int nSrcId = 0)
        {
            RawImageMean img = m_db.GetRawImageMean(nSrcId);
            return LoadDatum(img);
        }
 
        public bool CopyImageMean(string strSrcSrc, string strDstSrc)
        {
            int nSrcIdSrc = GetSourceID(strSrcSrc);
            int nDstIdSrc = GetSourceID(strDstSrc);
            return m_db.CopyImageMean(nSrcIdSrc, nDstIdSrc);
        }
 
        public int GetImageCount()
        {
            return m_db.GetImageCount();
        }
 
        public List<int> QueryRawImageIDs(int nSrcId = 0, int nMax = int.MaxValue, int nLabel = -1, int nBoost = -1, bool bBoostIsExact = false, bool bAnnotatedOnly = false)
        {
            return m_db.QueryAllRawImageIDs(nSrcId, nMax, nLabel, nBoost, bBoostIsExact, bAnnotatedOnly);
        }
 
        public int GetRawImageMeanID(int nSrcId = 0)
        {
            RawImageMean img = m_db.GetRawImageMean(nSrcId);
            return (img == null) ? 0 : img.ID;
        }
 
        public int GetRawImageID(DateTime dt, int nSrcId = 0)
        {
            return m_db.GetRawImageID(dt, nSrcId);
        }
 
        public RawImage GetRawImageFromID(int nImageID)
        {
            return m_db.GetRawImage(nImageID);
        }
 
        public void DeleteRawImageResults(int nSrcId = 0)
        {
            m_db.DeleteRawImageResults(nSrcId);
        }
 
        public int PutRawImageResults(int nSrcId, int nIdx, int nLabel, DateTime dt, List<Result> rgResults, bool bInvert, List<Tuple<DateTime, int>> rgExtra = null)
        {
            return m_db.PutRawImageResults(nSrcId, nIdx, nLabel, dt, rgResults, bInvert, rgExtra);
        }
 
        public int PutRawImageResults(int nSrcId, int nIdx, int nLabel, DateTime dt, List<Tuple<SimpleDatum, List<Result>>> rgrgResults, List<Tuple<DateTime, int>> rgExtra = null)
        {
            return m_db.PutRawImageResults(nSrcId, nIdx, nLabel, dt, rgrgResults, rgExtra);
        }
 
        public List<Tuple<SimpleDatum, List<Result>>> GetRawImageResultBatch(int nBatchCount, byte[] rgResults)
        {
            return m_db.GetRawImageResultBatch(nBatchCount, rgResults);
        }
 
        public int AddRawImageGroup(Image img, int nIdx, DateTime dtStart, DateTime dtEnd, List<double> rgProperties)
        {
            return m_db.AddRawImageGroup(img, nIdx, dtStart, dtEnd, rgProperties);
        }
 
        public int FindRawImageGroupID(int nIdx, DateTime dtStart, DateTime dtEnd)
        {
            return m_db.FindRawImageGroupID(nIdx, dtStart, dtEnd);
        }
 
        public List<string> GetRawImageDistinctParameterDescriptions(int nSrcId)
        {
            return m_db.GetRawImageDistinctParameterDescriptions(nSrcId);
        }
 
        public byte[] GetRawImageParameterData(int nRawImageID, string strParam)
        {
            return m_db.GetRawImageParameterData(nRawImageID, strParam);
        }
 
        public int GetRawImageParameterCount(string strParam, int nSrcId = 0, string strType = "TEXT")
        {
            return m_db.GetRawImageParameterCount(strParam, nSrcId, strType);
        }
 
        public bool GetRawImageParameterExist(string strName, int nSrcId = 0, string strType = "TEXT")
        {
            return m_db.GetRawImageParameterExist(strName, nSrcId, strType);
        }
 
        public bool UpdateActiveLabel(int nImageID, int nNewActiveLabel, bool bActivate = true, bool bSaveChanges = true)
        {
            return m_db.UpdateActiveLabel(nImageID, nNewActiveLabel, bActivate, bSaveChanges);
        }
 
        public void UpdateActiveLabelDirect(int nID, int nLabel)
        {
            m_db.UpdateActiveLabelDirect(nID, nLabel);
        }
 
        public void UpdateAllActiveLabelsDirect(int nLabel, int? nOriginalLabel = null)
        {
            m_db.UpdateAllActiveLabels(m_openSource.ID, nLabel, nOriginalLabel);
        }
 
        public void DisableLabel(int nLabel, bool bOriginalLabel = false)
        {
            m_db.DisableLabel(m_openSource.ID, nLabel, bOriginalLabel);
        }
 
        public void DisableAllLabels()
        {
            m_db.DisableAllLabels(m_openSource.ID);
        }
 
        public int DisableAllNonMatchingImages(int nWidth, int nHeight)
        {
            return m_db.DisableAllNonMatchingImages(m_openSource.ID, nWidth, nHeight);
        }
 
        public void ResetAllActiveLabels()
        {
            m_db.ResetAllActiveLabels(m_openSource.ID);
        }
 
        public void UpdateActiveLabelByID(int nID, int nNewActiveLabel)
        {
            m_db.UpdateActiveLabelByID(nID, nNewActiveLabel);
        }
 
        public void UpdateActiveLabelByIndex(int nIdx, int nNewActiveLabel)
        {
            m_db.UpdateActiveLabelByIndex(m_openSource.ID, nIdx, nNewActiveLabel);
        }
 
        public void ActivateRawImageByIndex(int nIdx, bool bActive)
        {
            m_db.ActivateRawImageByIndex(m_openSource.ID, nIdx, bActive);
        }
 
        public void UpdateRawImageSourceID(int nImageID, int nSrcID)
        {
            m_db.UpdateRawImageSourceID(nImageID, nSrcID);
        }
 
        public bool ActivateRawImage(int nImageID, bool bActive, bool bSave = true)
        {
            return m_db.ActivateRawImage(nImageID, bActive, bSave);
        }
 
        public void ActivateAllRawImages(bool bActive, bool bAnnotatedOnly, params int[] rgSrcId)
        {
            m_db.ActivateAllRawImages(bActive, bAnnotatedOnly, rgSrcId);
        }
 
        public void ActivateAllRawImages(bool bActive, bool bAnnotatedOnly, int? nTgtLabel, bool bTargetLabelExact, int? nTgtBoost, bool bTargetBoostExact, params int[] rgSrcId)
        {
            m_db.ActivateAllRawImages(bActive, bAnnotatedOnly, nTgtLabel, bTargetLabelExact, nTgtBoost, bTargetBoostExact, rgSrcId);
        }
 
        public void FixupRawImageCopy(int nImageID, int nSecondarySrcId)
        {
            m_db.FixupRawImageCopy(nImageID, nSecondarySrcId);
        }
 
        public byte[] GetRawImageDataCriteria(byte[] rgData, int nOriginalSourceID)
        {
            return m_db.GetRawImageDataCriteria(rgData, nOriginalSourceID);
        }
 
        public byte[] GetRawImageDataCriteria(int nImgID, int nOriginalSourceID)
        {
            return m_db.GetRawImageDataCriteria(nImgID, nOriginalSourceID);
        }
 
 
        public byte[] GetRawImageDebugData(byte[] rgData, int nOriginalSourceID)
        {
            return m_db.GetRawImageDebugData(rgData, nOriginalSourceID);
        }
 
        public byte[] GetRawImageDebugData(int nImgID, int nOriginalSourceID)
        {
            return m_db.GetRawImageDebugData(nImgID, nOriginalSourceID);
        }
 
        public void UpdateDatasetImageAnnotations(int nSrcId, int nImageId, AnnotationGroupCollection annotations, bool bSetLabelOnly)
        {
            m_db.UpdateDatasetImageAnnotations(nSrcId, nImageId, annotations, bSetLabelOnly);
        }
 
        #endregion
 
 
        //---------------------------------------------------------------------
        //  Labels
        //---------------------------------------------------------------------
        #region Labels
 
        public List<LabelBoostDescriptor> GetLabelBoosts(int nProjectId, int nSrcId = 0)
        {
            List<LabelBoost> rgBoosts = m_db.GetLabelBoosts(nProjectId, true, nSrcId);
            List<LabelBoostDescriptor> rgDesc = new List<LabelBoostDescriptor>();
 
            foreach (LabelBoost b in rgBoosts)
            {
                rgDesc.Add(new LabelBoostDescriptor(b.ActiveLabel.GetValueOrDefault(), (double)b.Boost.GetValueOrDefault(1)));
            }
 
            return rgDesc;
        }
 
        public void SetLabelMapping(LabelMapping map, int nSrcId = 0)
        {
            m_db.SetLabelMapping(map, nSrcId);
        }
 
        public void UpdateLabelMapping(int nNewLabel, List<int> rgOriginalLabels, int nSrcId = 0)
        {
            m_db.UpdateLabelMapping(nNewLabel, rgOriginalLabels, nSrcId);
        }
 
        public void ResetLabels(int nProjectId = 0, int nSrcId = 0)
        {
            m_db.ResetLabels(nProjectId, nSrcId);
        }
 
        public void DeleteLabelBoosts(int nProjectId, int nSrcId = 0)
        {
            m_db.DeleteLabelBoosts(nProjectId, nSrcId);
        }
 
        public void AddLabelBoost(int nProjectId, int nLabel, double dfBoost, int nSrcId = 0)
        {
            m_db.AddLabelBoost(nProjectId, nLabel, dfBoost, nSrcId);
        }
 
        public string GetLabelBoostsAsText(int nProjectId, int nSrcId = 0, bool bSort = true)
        {
            return m_db.GetLabelBoostsAsText(nProjectId, nSrcId, bSort);
        }
 
        public Dictionary<int, int> LoadLabelCounts(int nSrcId = 0)
        {
            Dictionary<int, int> rgCounts = new Dictionary<int, int>();
            m_db.LoadLabelCounts(rgCounts, nSrcId);
            return rgCounts;
        }
 
        public void UpdateLabelCounts(Dictionary<int, int> rgCounts)
        {
            m_db.UpdateLabelCounts(rgCounts);
        }
 
        public void UpdateLabelCounts(int nSrcId = 0, int nProjectId = 0)
        {
            m_db.UpdateLabelCounts(nSrcId, nProjectId);
        }
 
        public string GetLabelCountsAsText(int nSrcId)
        {
            return m_db.GetLabelCountsAsText(nSrcId);
        }
 
        public void UpdateLabelName(int nLabel, string strName, int nSrcId = 0)
        {
            m_db.UpdateLabelName(nLabel, strName, nSrcId);
        }
 
        public string GetLabelName(int nLabel, int nSrcId = 0)
        {
            return m_db.GetLabelName(nLabel, nSrcId);
        }
 
        public int AddLabel(int nLabel, string strName, int nSrcId = 0, ConnectInfo ci = null)
        {
            return m_db.AddLabel(nLabel, strName, nSrcId, ci);
        }
 
        public void DeleteLabels(int nSrcId = 0)
        {
            m_db.DeleteLabels(nSrcId);
        }
 
        public void ActivateLabels(List<int> rgLabels, bool bActive, params int[] rgSrcId)
        {
            m_db.ActivateLabels(rgLabels, bActive, rgSrcId);
        }
 
        public void UpdateLabelBoost(int? nTgtLbl, bool bTgtLblExact, int? nTgtBst, bool bTgtBstExact, int? nNewLbl, int? nNewBst, params int[] rgSrcId)
        {
            m_db.UpdateLabelBoost(nTgtLbl, bTgtLblExact, nTgtBst, bTgtBstExact, nNewLbl, nNewBst, rgSrcId);
        }
 
        public List<Label> QueryLabels(int nSrcId = 0, ConnectInfo ci = null)
        {
            return m_db.GetLabels(false, false, nSrcId, ci);
        }
 
        #endregion
 
 
        //---------------------------------------------------------------------
        //  Sources
        //---------------------------------------------------------------------
        #region Sources
 
        public int GetSourceID(string strName)
        {
            return m_db.GetSourceID(strName);
        }
 
        public string GetSourceName(int nId)
        {
            return m_db.GetSourceName(nId);
        }
 
        public int AddSource(SourceDescriptor src, ConnectInfo ci = null, bool? bSaveImagesToFileOverride = null)
        {
            bool bSaveImagesToFile = bSaveImagesToFileOverride.GetValueOrDefault(src.SaveImagesToFile);
            src.ID = m_db.AddSource(src.Name, src.Channels, src.Width, src.Height, src.IsRealData, src.CopyOfSourceID, bSaveImagesToFile, ci);
            return src.ID;
        }
 
        public int AddSource(string strName, int nChannels, int nWidth, int nHeight, bool bDataIsReal, int nCopyOfSourceID = 0, bool bSaveImagesToFile = true, ConnectInfo ci = null)
        {
            return m_db.AddSource(strName, nChannels, nWidth, nHeight, bDataIsReal, nCopyOfSourceID, bSaveImagesToFile, ci);
        }
 
        public void DeleteSources(params string[] rgstrSrc)
        {
            m_db.DeleteSources(rgstrSrc);
        }
 
        public void DeleteSourceData(int nSrcId = 0)
        {
            m_db.DeleteSourceData(nSrcId);
        }
 
        public int GetBoostCount(int nSrcId = 0, string strFilterVal = null, int? nBoostVal = null)
        {
            return m_db.GetBoostCount(nSrcId, strFilterVal, nBoostVal);
        }
 
        public int ActivateFiltered(int nSrcId = 0, string strFilterVal = null, int? nBoostVal = null)
        {
            return m_db.ActivateFiltered(nSrcId, strFilterVal, nBoostVal);
        }
 
        public bool ReindexRawImages(Log log, CancelEvent evtCancel, int nSrcId = 0, bool bCreateImageMean = false)
        {
            List<RawImage> rgImg = m_db.ReindexRawImages(log, evtCancel, nSrcId);
            if (rgImg == null)
                return false;
 
            if (bCreateImageMean)
            {
                bool bOpened = false;
                if (m_openSource == null)
                {
                    Open(nSrcId);
                    bOpened = true;
                }
 
                List<SimpleDatum> rgSd = new List<SimpleDatum>();
                foreach (RawImage img in rgImg)
                {
                    rgSd.Add(LoadDatum(img));
                }
 
                SimpleDatum sdMean = SimpleDatum.CalculateMean(log, rgSd.ToArray(), evtCancel.Handles);
                if (sdMean == null)
                    return false;
 
                bool bRes = SaveImageMean(sdMean, true, nSrcId);
                if (bOpened)
                    Close();
 
                return bRes;
            }
 
            return true;
        }
 
        public void UpdateSource(int nChannels, int nWidth, int nHeight, bool bDataIsReal, int nSrcId = 0)
        {
            m_db.UpdateSource(nChannels, nWidth, nHeight, bDataIsReal, nSrcId);
        }
 
        public void UpdateSourceCounts()
        {
            m_db.SaveLabelCache();
            m_db.UpdateLabelCounts();
            m_db.UpdateSourceCounts();
        }
 
        public void SetSourceParameter(string strParam, string strValue, int nSrcId = 0)
        {
            m_db.SetSourceParameter(strParam, strValue, nSrcId);
        }
 
        public string GetSourceParameter(string strParam, int nSrcId = 0)
        {
            return m_db.GetSourceParameter(strParam, nSrcId);
        }
 
        public int GetSourceParameter(string strParam, int nDefault, int nSrcId = 0)
        {
            return m_db.GetSourceParameter(strParam, nDefault, nSrcId);
        }
 
        public bool GetSourceParameter(string strParam, bool bDefault, int nSrcId = 0)
        {
            return m_db.GetSourceParameter(strParam, bDefault, nSrcId);
        }
 
        public double GetSourceParameter(string strParam, double dfDefault, int nSrcId = 0)
        {
            return m_db.GetSourceParameter(strParam, dfDefault, nSrcId);
        }
 
        public DateTime GetFirstTimeStamp(int nSrcId = 0, string strDesc = null)
        {
            return m_db.GetFirstTimeStamp(nSrcId, strDesc);
        }
 
        public DateTime GetLastTimeStamp(int nSrcId = 0, string strDesc = null)
        {
            return m_db.GetLastTimeStamp(nSrcId, strDesc);
        }
 
        public DateTime GetLastTimeStamp(DateTime dtStart, DateTime dtEnd, bool bEndInclusive, int nSrcId = 0, string strDesc = null)
        {
            return m_db.GetLastTimeStamp(dtStart, dtEnd, bEndInclusive, nSrcId, strDesc);
        }
 
 
        public DateTime GetLastTimeStamp(out int nIndex, int nSrcId = 0, string strDesc = null)
        {
            return m_db.GetLastTimeStamp(out nIndex, nSrcId, strDesc);
        }
 
        public DateTime GetLastTimeStamp(DateTime dtStart, DateTime dtEnd, bool bEndInclusive, out int nIndex, int nSrcId = 0, string strDesc = null)
        {
            return m_db.GetLastTimeStamp(dtStart, dtEnd, bEndInclusive, out nIndex);
        }
 
#pragma warning disable 1591
 
        public List<int> GetAllDataSourceIDs() 
        {
            return m_db.GetAllDataSourcesIDs();
        }
 
        public bool ConvertRawImagesSaveToFile(int nIdx, int nCount) 
        {
            return m_db.ConvertRawImagesSaveToFile(nIdx, nCount);
        }
 
        public bool ConvertRawImagesSaveToDatabase(int nIdx, int nCount) 
        {
            return m_db.ConvertRawImagesSaveToDatabase(nIdx, nCount);
        }
 
        public void UpdateSaveImagesToFile(bool bSaveToFile, int nSrcId = 0) 
        {
            m_db.UpdateSaveImagesToFile(bSaveToFile, nSrcId);
        }
 
#pragma warning restore 1591
 
        public int? OriginalSourceID
        {
            get { return m_nOriginalSourceID; }
            set { m_nOriginalSourceID = value; }
        }
 
        #endregion
 
 
        //---------------------------------------------------------------------
        //  Datasets
        //---------------------------------------------------------------------
        #region Datasets
 
        public int GetDatasetID(string strDsName, ConnectInfo ci = null)
        {
            return m_db.GetDatasetID(strDsName, ci);
        }
 
        public string GetDatasetName(int nId, ConnectInfo ci = null)
        {
            return m_db.GetDatasetName(nId, ci);
        }
 
        public int AddDataset(DatasetDescriptor ds, ConnectInfo ci = null, bool? bSaveImagesToFileOverride = null)
        {
            ds.TestingSource.ID = AddSource(ds.TestingSource, ci);
            ds.TrainingSource.ID = AddSource(ds.TrainingSource, ci);
 
            int nDsCreatorID = 0;
            if (ds.CreatorName != null)
                nDsCreatorID = m_db.GetDatasetCreatorID(ds.CreatorName);
 
            ds.ID = m_db.AddDataset(nDsCreatorID, ds.Name, ds.TestingSource.ID, ds.TrainingSource.ID, ds.DatasetGroup.ID, ds.ModelGroup.ID, ci);
            return ds.ID;
        }
 
        public int AddDataset(int nDsCreatorID, string strName, int nTestSrcId, int nTrainSrcId, int nDsGroupID = 0, int nModelGroupID = 0, ConnectInfo ci = null)
        {
            return m_db.AddDataset(nDsCreatorID, strName, nTestSrcId, nTrainSrcId, nDsGroupID, nModelGroupID, ci);
        }
 
        public void UpdateDatasetDescription(int nDsId, string strDesc)
        {
            m_db.UpdateDatasetDescription(nDsId, strDesc);
        }
 
        public void UpdateDatasetCounts(int nDsId, ConnectInfo ci = null)
        {
            Dataset ds = m_db.GetDataset(nDsId, ci);
            Database db = new Database();
 
            db.Open(ds.TestingSourceID.GetValueOrDefault(), Database.FORCE_LOAD.NONE, ci);
            db.UpdateSourceCounts(ci);
            db.UpdateLabelCounts(0, 0, ci);
            db.Close();
 
            db.Open(ds.TrainingSourceID.GetValueOrDefault(), Database.FORCE_LOAD.NONE, ci);
            db.UpdateSourceCounts(ci);
            db.UpdateLabelCounts(0, 0, ci);
            db.Close();
 
            m_db.UpdateDatasetCounts(nDsId, ci);
        }
 
        public void UpdateDatasetCounts(CancelEvent evtCancel, Log log, int nDatasetCreatorID, List<string> rgstrDs, string strParamNameForDescription)
        {
            m_db.UpdateDatasetCounts(evtCancel, log, nDatasetCreatorID, rgstrDs, strParamNameForDescription);
        }
 
        public string FindDatasetNameFromSourceName(string strTrainSrc, string strTestSrc)
        {
            return m_db.FindDatasetNameFromSourceName(strTrainSrc, strTestSrc);
        }
 
        public int FindDatasetFromSourceId(int nSourceId)
        {
            return m_db.FindDatasetFromSourceId(nSourceId);
        }
 
        public void ResetAllDatasetRelabelWithCreator(int nDsCreatorId)
        {
            m_db.ResetAllDatasetRelabelWithCreator(nDsCreatorId);
        }
 
        public void UpdateDatasetRelabel(int nDsId, bool bRelabel)
        {
            m_db.UpdateDatasetRelabel(nDsId, bRelabel);
        }
 
        public void SetDatasetParameters(int nDsId, Dictionary<string, string> rgP)
        {
            m_db.SetDatasetParameters(nDsId, rgP);
        }
 
        public void SetDatasetParameter(int nDsId, string strParam, string strValue)
        {
            m_db.SetDatasetParameter(nDsId, strParam, strValue);
        }
 
        public string GetDatasetParameter(int nDsId, string strParam)
        {
            return m_db.GetDatasetParameter(nDsId, strParam);
        }
 
        public int GetDatasetParameter(int nDsId, string strParam, int nDefault)
        {
            return m_db.GetDatasetParameter(nDsId, strParam, nDefault);
        }
 
        public bool GetDatasetParameter(int nDsId, string strParam, bool bDefault)
        {
            return m_db.GetDatasetParameter(nDsId, strParam, bDefault);
        }
 
        public double GetDatasetParameter(int nDsId, string strParam, double dfDefault)
        {
            return m_db.GetDatasetParameter(nDsId, strParam, dfDefault);
        }
 
        public int GetDatasetGroupID(string strName)
        {
            return m_db.GetDatasetGroupID(strName);
        }
 
        #endregion
 
        //---------------------------------------------------------------------
        //  Results
        //---------------------------------------------------------------------
        #region Results
 
        public List<SimpleResult> QueryAllResults(int nSrcID, bool bRequireExtraData = false, int nMax = -1)
        {
            List<RawImageResult> rgRes1 = m_db.GetRawImageResults(nSrcID, bRequireExtraData, nMax);
            List<SimpleResult> rgRes = new List<SimpleResult>();
 
            foreach (RawImageResult res1 in rgRes1)
            {
                SimpleResult res = LoadResult(res1, bRequireExtraData);
                if (res != null)
                    rgRes.Add(res);
            }
 
            return rgRes;
        }
 
        #endregion // Results
 
 
        //---------------------------------------------------------------------
        //  Loading Descriptors
        //---------------------------------------------------------------------
        #region Loading Descriptors
 
        public List<DbItem> LoadImageIndexes(bool bBoostedOnly, bool bIncludeActive = true, bool bIncludeInactive = false)
        {
            return m_db.GetAllRawImageIndexes(bBoostedOnly, bIncludeActive, bIncludeInactive);
        }
 
        public List<ImageDescriptor> LoadImages(CancelEvent evtCancel, params int[] rgSrcId)
        {
            if (rgSrcId.Length == 0)
                throw new Exception("You must specify at least one source ID.");
 
            List<ImageDescriptor> rgImgDesc = new List<ImageDescriptor>();
            List<RawImage> rgImg = m_db.QueryRawImages(rgSrcId);
 
            foreach (RawImage img in rgImg)
            {
                if (evtCancel != null && evtCancel.WaitOne(0))
                    return null;
 
                rgImgDesc.Add(new ImageDescriptor(img.ID,
                                                  img.Height.GetValueOrDefault(0),
                                                  img.Width.GetValueOrDefault(0),
                                                  img.Channels.GetValueOrDefault(0),
                                                  img.Encoded.GetValueOrDefault(false),
                                                  img.SourceID.GetValueOrDefault(0),
                                                  img.Idx.GetValueOrDefault(0),
                                                  img.ActiveLabel.GetValueOrDefault(0),
                                                  img.Active.GetValueOrDefault(false),
                                                  img.Description,
                                                  img.TimeStamp.GetValueOrDefault(DateTime.MinValue)));
            }
 
            return rgImgDesc;
        }
 
        public SimpleDatum LoadDatum(int nImageId, int nChannels, bool bDataIsReal, int nLabel, int nSrcId = 0)
        {
            RawImage img = m_db.GetRawImage(nImageId);
 
            if (img == null)
                return null;
 
            if (nSrcId == 0)
                nSrcId = m_openSource.ID;
 
            if (img.SourceID != nSrcId)
                throw new Exception("The source of the raw image with ID = " + nImageId.ToString() + " (SourceID = " + img.SourceID.ToString() + ") does not match the expected source id of " + nSrcId.ToString());
 
            if (img.Channels != nChannels)
                throw new Exception("The raw image with ID = " + nImageId.ToString() + " has a different channel count than the expected count of " + nChannels.ToString());
 
            if (img.Encoded.GetValueOrDefault() != bDataIsReal)
                throw new Exception("The raw image with ID = " + nImageId.ToString() + " does not have a matching encoding scheme as that of the expected encoded = " + bDataIsReal.ToString());
 
            if (img.ActiveLabel != (int)nLabel)
                throw new Exception("The raw image with ID = " + nImageId.ToString() + " does not have a label equal to the expected label of " + nLabel.ToString());
 
            return new Datum(LoadDatum(img));
        }
 
        public SimpleDatum LoadDatum(RawImage img, int nPadW = 0, int nPadH = 0)
        {
            if (img == null)
                return null;
 
            byte[] rgDataCriteria = null;
            int? nDataCriteriaFormatId = null;
            byte[] rgDebugData = null;
            int? nDebugDataFormatId = null;
            byte[] rgData = m_db.GetRawImageData(img, m_bLoadDataCriteria, m_bLoadDebugData, out rgDataCriteria, out nDataCriteriaFormatId, out rgDebugData, out nDebugDataFormatId);
 
            // Annotation Data is used as the label, so it must be loaded.
            if (!m_bLoadDataCriteria &&
                (nDataCriteriaFormatId.GetValueOrDefault(0) == (int)SimpleDatum.DATA_FORMAT.SEGMENTATION ||
                 nDataCriteriaFormatId.GetValueOrDefault(0) == (int)SimpleDatum.DATA_FORMAT.ANNOTATION_DATA))
            {
                LoadRawImageData(img, true, false);
                rgDataCriteria = img.DataCriteria;
            }
 
            int nHeight = img.Height.GetValueOrDefault();
            int nWidth = img.Width.GetValueOrDefault();
            int nChannels = img.Channels.GetValueOrDefault();
            SimpleDatum sd = null;
 
            if (img.Encoded.GetValueOrDefault())
            {
                Tuple<double[], float[]> rgRealData = SimpleDatum.GetRealData(rgData, nPadW, nPadH, nHeight, nWidth, nChannels);
                double[] rgDataFloat = rgRealData.Item1;
                float[] rgDataDouble = rgRealData.Item2;
 
                if (rgDataFloat != null)
                {
                    sd = new SimpleDatum(img.Encoded.GetValueOrDefault(),
                                           nChannels,
                                           nWidth + nPadW,
                                           nHeight + nPadH,
                                           img.ActiveLabel.GetValueOrDefault(),
                                           img.TimeStamp.GetValueOrDefault(),
                                           rgDataFloat,
                                           img.ActiveBoost.GetValueOrDefault(),
                                           img.AutoLabel.GetValueOrDefault(),
                                           img.Idx.GetValueOrDefault(),
                                           img.VirtualID.GetValueOrDefault(),
                                           img.ID,
                                           img.SourceID.GetValueOrDefault(),
                                           img.OriginalSourceID.GetValueOrDefault());
                }
                else
                {
                    sd = new SimpleDatum(img.Encoded.GetValueOrDefault(),
                                           nChannels,
                                           nWidth + nPadW,
                                           nHeight + nPadH,
                                           img.ActiveLabel.GetValueOrDefault(),
                                           img.TimeStamp.GetValueOrDefault(),
                                           rgDataDouble,
                                           img.ActiveBoost.GetValueOrDefault(),
                                           img.AutoLabel.GetValueOrDefault(),
                                           img.Idx.GetValueOrDefault(),
                                           img.VirtualID.GetValueOrDefault(),
                                           img.ID,
                                           img.SourceID.GetValueOrDefault(),
                                           img.OriginalSourceID.GetValueOrDefault());
                }
            }
            else
            {
                byte[] rgDataBytes = SimpleDatum.GetByteData(rgData, nPadW, nPadH, nHeight, nWidth, nChannels);
                sd = new SimpleDatum(img.Encoded.GetValueOrDefault(),
                                   nChannels,
                                   nWidth + nPadW,
                                   nHeight + nPadH,
                                   img.ActiveLabel.GetValueOrDefault(),
                                   img.TimeStamp.GetValueOrDefault(),
                                   rgDataBytes,                                   
                                   img.ActiveBoost.GetValueOrDefault(),
                                   img.AutoLabel.GetValueOrDefault(),
                                   img.Idx.GetValueOrDefault(),
                                   img.VirtualID.GetValueOrDefault(),
                                   img.ID,
                                   img.SourceID.GetValueOrDefault(),
                                   img.OriginalSourceID.GetValueOrDefault());
            }
 
            sd.OriginalLabel = img.OriginalLabel.GetValueOrDefault();
            sd.Description = img.Description;
            sd.GroupID = img.GroupID.GetValueOrDefault();
            sd.DataCriteria = rgDataCriteria;
            sd.DataCriteriaFormat = (SimpleDatum.DATA_FORMAT)nDataCriteriaFormatId.GetValueOrDefault(0);
            sd.DebugData = rgDebugData;
            sd.DebugDataFormat = (SimpleDatum.DATA_FORMAT)nDebugDataFormatId.GetValueOrDefault(0);
 
            sd.LoadAnnotationDataFromDataCriteria();
 
            return sd;
        }
 
        public SimpleDatum LoadDatum(RawImageMean img, int nPadW = 0, int nPadH = 0)
        {
            if (img == null)
                return null;
 
            int nHeight = img.Height.GetValueOrDefault();
            int nWidth = img.Width.GetValueOrDefault();
            int nChannels = img.Channels.GetValueOrDefault();
            SimpleDatum sd;
 
            if (img.Encoded.GetValueOrDefault())
            {
                Tuple<double[], float[]> rgRealData = SimpleDatum.GetRealData(img.Data, nPadW, nPadH, nHeight, nWidth, nChannels);
                double[] rgDataFloat = rgRealData.Item1;
                float[] rgDataDouble = rgRealData.Item2;
 
                if (rgDataFloat != null)
                {
                    sd = new SimpleDatum(img.Encoded.GetValueOrDefault(),
                                                  nChannels,
                                                  nWidth + nPadW,
                                                  nHeight + nPadH,
                                                  0,
                                                  DateTime.MinValue,
                                                  rgDataFloat,
                                                  0,
                                                  false,
                                                  0,
                                                  0,
                                                  img.ID,
                                                  img.SourceID.GetValueOrDefault());
                }
                else
                {
                    sd = new SimpleDatum(img.Encoded.GetValueOrDefault(),
                                                  nChannels,
                                                  nWidth + nPadW,
                                                  nHeight + nPadH,
                                                  0,
                                                  DateTime.MinValue,
                                                  rgDataDouble,
                                                  0,
                                                  false,
                                                  0,
                                                  0,
                                                  img.ID,
                                                  img.SourceID.GetValueOrDefault());
                }
            }
            else
            {
                byte[] rgDataBytes = SimpleDatum.GetByteData(img.Data, nPadW, nPadH, nHeight, nWidth, nChannels);
                sd = new SimpleDatum(img.Encoded.GetValueOrDefault(),
                                              nChannels,
                                              nWidth + nPadW,
                                              nHeight + nPadH,
                                              0,
                                              DateTime.MinValue,
                                              rgDataBytes,
                                              0,
                                              false,
                                              0,
                                              0,
                                              img.ID,
                                              img.SourceID.GetValueOrDefault());
            }
 
            return sd;
        }
 
        public SimpleDatum LoadImage(int nImageId, int nSrcId = 0)
        {
            if (m_db.CurrentSource == null)
                throw new Exception("You must open a data source first!");
 
            if (nSrcId == 0)
                nSrcId = m_db.CurrentSource.ID;
 
            RawImage img = m_db.GetRawImage(nImageId);
            if (img == null || img.SourceID != nSrcId)
                return null;
 
            return LoadDatum(img);
        }
 
        public SimpleDatum LoadImageMean(int nSrcId, ConnectInfo ci = null)
        {
            RawImageMean imgMean = m_db.GetRawImageMean(nSrcId, ci);
            if (imgMean == null)
                return null;
 
            return LoadDatum(imgMean);
        }
 
        public SimpleDatum LoadImageAt(int nIdx, bool? bLoadDataCriteria = null, bool? bLoadDebugData = null, int nSrcId = 0, int nPadW = 0, int nPadH = 0)
        {            
            RawImage img = m_db.GetRawImageAt(nIdx, nSrcId);
            if (img == null)
                return null;
 
            if (!bLoadDataCriteria.HasValue)
                bLoadDataCriteria = m_bLoadDataCriteria;
 
            if (bLoadDataCriteria.Value && img.DataCriteria != null)
                img.DataCriteria = m_db.GetRawImageDataCriteria(img.DataCriteria);
 
            if (!bLoadDebugData.HasValue)
                bLoadDebugData = m_bLoadDebugData;
 
            if (bLoadDebugData.Value && img.DebugData != null)
                img.DebugData = m_db.GetRawImageDebugData(img.DebugData);
 
            return LoadDatum(img, nPadW, nPadH);
        }
 
        public SimpleResult LoadResult(RawImageResult res, bool bRequireExtraData = false)
        {
            int nSrcID = res.SourceID.GetValueOrDefault();
            DateTime dt = res.TimeStamp.GetValueOrDefault();
            int nIdx = res.Idx.GetValueOrDefault();
 
            int nResCount = res.ResultCount.GetValueOrDefault();
            int nBatchCount = res.BatchCount.GetValueOrDefault();
            List<Tuple<SimpleDatum, List<Result>>> rgRes = ResultDescriptor.GetResults(nBatchCount, res.Results);
            List<float> rgResults = new List<float>();
 
            for (int i = 0; i < rgRes.Count; i++)
            {
                for (int j = 0; j < rgRes[i].Item2.Count; j++)
                {
                    rgResults.Add((float)rgRes[i].Item2[j].Score);
                }
            }
 
            float[] rgResult1 = rgResults.ToArray();
            List<Tuple<DateTime, int>> rgTarget1 = new List<Tuple<DateTime, int>>();
 
            if (res.ExtraData == null && bRequireExtraData)
                return null;
 
            rgTarget1 = Database.UnpackExtraData(res.ExtraData);
 
            return new SimpleResult(nSrcID, nIdx, dt, nBatchCount, nResCount, rgResult1, rgTarget1);
        }
 
        public void LoadRawData(SimpleDatum sd, bool bLoadDataCriteria, bool bLoadDebugData)
        {
            if (bLoadDataCriteria && sd.DataCriteria != null)
                sd.DataCriteria = m_db.GetRawImageDataCriteria(sd.DataCriteria, sd.OriginalSourceID);
 
            if (bLoadDebugData && sd.DebugData != null)
                sd.DebugData = m_db.GetRawImageDebugData(sd.DebugData, sd.OriginalSourceID);
        }
 
        public List<LabelDescriptor> LoadLabels(int nSrcId = 0, bool bSort = true, bool bWithImagesOnly = false, ConnectInfo ci = null)
        {
            List<Label> rgLabels = m_db.GetLabels(bSort, bWithImagesOnly, nSrcId, ci);
            List<LabelDescriptor> rgDesc = new List<LabelDescriptor>();
 
            foreach (Label l in rgLabels)
            {
                LabelDescriptor label = new LabelDescriptor(l.Label1.GetValueOrDefault(),
                                                            l.ActiveLabel.GetValueOrDefault(),
                                                            l.Name,
                                                            l.ImageCount.GetValueOrDefault());
                rgDesc.Add(label);
            }
 
            return rgDesc;
        }
 
        public SourceDescriptor LoadSource(string strSource)
        {
            return LoadSource(m_db.GetSourceID(strSource));
        }
 
        public SourceDescriptor LoadSource(int nSrcId, ConnectInfo ci = null)
        {
            Source src = m_db.GetSource(nSrcId, ci);
 
            if (src == null)
                return null;
 
            SourceDescriptor srcDesc = new SourceDescriptor(src.ID,
                                                            src.Name,
                                                            src.ImageHeight.GetValueOrDefault(),
                                                            src.ImageWidth.GetValueOrDefault(),
                                                            src.ImageChannels.GetValueOrDefault(),
                                                            src.ImageEncoded.GetValueOrDefault(),
                                                            src.SaveImagesToFile.GetValueOrDefault(),
                                                            src.CopyOfSourceID.GetValueOrDefault(0),
                                                            src.OwnerID,
                                                            src.ImageCount.GetValueOrDefault());
            srcDesc.Labels = LoadLabels(nSrcId, true, false, ci);
            srcDesc.LabelCountsAsText = m_db.GetLabelCountsAsText(nSrcId, ci);
            srcDesc.SetInactiveImageCount(m_db.GetImageCount(nSrcId, false, true, ci));
            srcDesc.Parameters = LoadSourceParameters(srcDesc.ID, ci);
 
            return srcDesc;
        }
 
        public ParameterDescriptorCollection LoadSourceParameters(int nSrcId, ConnectInfo ci = null)
        {
            ParameterDescriptorCollection col = new ParameterDescriptorCollection();
 
            Dictionary<string, string> rgParam = m_db.GetSourceParameters(nSrcId, ci);
            foreach (KeyValuePair<string, string> kv in rgParam)
            {
                col.Add(new ParameterDescriptor(0, kv.Key, kv.Value));
            }
 
            return col;
        }
 
        public GroupDescriptor LoadDatasetGroup(int nGroupId, ConnectInfo ci = null)
        {
            DatasetGroup grp = m_db.GetDatasetGroup(nGroupId, ci);
 
            if (grp == null)
                return null;
 
            return new GroupDescriptor(nGroupId, grp.Name, grp.OwnerID);
        }
 
        public GroupDescriptor LoadModelGroup(int nGroupId, ConnectInfo ci = null)
        {
            ModelGroup grp = m_db.GetModelGroup(nGroupId, ci);
 
            if (grp == null)
                return null;
 
            return new GroupDescriptor(nGroupId, grp.Name, grp.OwnerID);
        }
 
        public DatasetDescriptor LoadDataset(string strDataset, ConnectInfo ci = null)
        {
            return LoadDataset(m_db.GetDatasetID(strDataset, ci), ci);
        }
 
        public DatasetDescriptor LoadDataset(int nDatasetID, ConnectInfo ci = null)
        {
            Dataset ds = m_db.GetDataset(nDatasetID, ci);
            if (ds == null)
                return null;
 
            return loadDataset(ds, ci);
        }
 
        public DatasetDescriptor LoadDataset(int nDatasetID, string strDataset)
        {
            DatasetDescriptor ds = null;
 
            if (nDatasetID != 0)
                ds = LoadDataset(nDatasetID);
 
            if (ds == null && !String.IsNullOrEmpty(strDataset))
                ds = LoadDataset(strDataset);
 
            return ds;
        }
 
        public DatasetDescriptor LoadDataset(string strTestingSrc, string strTrainingSrc)
        {
            Dataset ds = m_db.GetDataset(strTestingSrc, strTrainingSrc);
            if (ds == null)
                return null;
 
            return loadDataset(ds);
        }
 
        public List<DatasetDescriptor> LoadAllDatasetsWithCreators(int nGroupId)
        {
            List<Dataset> rgDs = m_db.GetAllDatasetsWithCreators(nGroupId);
            List<DatasetDescriptor> rgDesc = new List<DatasetDescriptor>();
 
            foreach (Dataset ds in rgDs)
            {
                rgDesc.Add(loadDataset(ds));
            }
 
            return rgDesc;
        }
 
        public List<DatasetDescriptor> LoadAllDatasetsWithCreator(int nCreatorID, bool? bRelabeled = null)
        {
            List<Dataset> rgDs = m_db.GetAllDatasetsWithCreator(nCreatorID, bRelabeled);
            List<DatasetDescriptor> rgDesc = new List<DatasetDescriptor>();
 
            foreach (Dataset ds in rgDs)
            {
                rgDesc.Add(LoadDataset(ds.ID));
            }
 
            return rgDesc;
        }
 
        protected virtual DatasetDescriptor loadDataset(Dataset ds, ConnectInfo ci = null)
        {
            SourceDescriptor srcTrain = LoadSource(ds.TrainingSourceID.GetValueOrDefault(), ci);
            SourceDescriptor srcTest = LoadSource(ds.TestingSourceID.GetValueOrDefault(), ci);
            GroupDescriptor dsGroup = LoadDatasetGroup(ds.DatasetGroupID.GetValueOrDefault(), ci);
            GroupDescriptor mdlGroup = LoadModelGroup(ds.ModelGroupID.GetValueOrDefault(), ci);
            string strCreatorName = m_db.GetDatasetCreatorName(ds.DatasetCreatorID.GetValueOrDefault(), ci);
            DatasetDescriptor dsDesc = new DatasetDescriptor(ds.ID, ds.Name, mdlGroup, dsGroup, srcTrain, srcTest, strCreatorName, ds.OwnerID, ds.Description);
 
            dsDesc.Parameters = LoadDatasetParameters(ds.ID, ci);
 
            return dsDesc;
        }
 
        public ParameterDescriptorCollection LoadDatasetParameters(int nDsId, ConnectInfo ci = null)
        {
            ParameterDescriptorCollection col = new ParameterDescriptorCollection();
 
            Dictionary<string, string> rgParam = m_db.GetDatasetParameters(nDsId, ci);
            foreach (KeyValuePair<string, string> kv in rgParam)
            {
                col.Add(new ParameterDescriptor(0, kv.Key, kv.Value));
            }
 
            return col;
        }
 
        public List<ResultDescriptor> LoadRawImageResults(int nSrcId = 0)
        {
            List<RawImageResult> rgResults = m_db.GetRawImageResults(nSrcId);
            List<ResultDescriptor> rgDesc = new List<ResultDescriptor>();
 
            foreach (RawImageResult res in rgResults)
            {
                rgDesc.Add(new ResultDescriptor(res.ID, null, null, res.Idx.GetValueOrDefault(), res.Label.GetValueOrDefault(), res.ResultCount.GetValueOrDefault(), res.Results, res.SourceID.GetValueOrDefault(), res.TimeStamp.GetValueOrDefault()));
            }
 
            return rgDesc;
        }
 
        #endregion
    }
 
    class ParamCache 
    {
        List<ParameterData> m_rgParam = new List<ParameterData>();
        int m_nMax;
 
        public ParamCache(int nMax)
        {
            m_nMax = nMax;
        }
 
        public int Count
        {
            get { return m_rgParam.Count; }
        }
 
        public void Clear()
        {
            m_rgParam.Clear();
        }
 
        public bool Add(ParameterData p)
        {
            m_rgParam.Add(p);
 
            if (m_rgParam.Count == m_nMax)
                return true;
 
            return false;
        }
 
        public List<ParameterData> Parameters
        {
            get { return m_rgParam; }
        }
    }
 
    class ImageCache 
    {
        List<RawImage> m_rgImages = new List<RawImage>();
        List<List<ParameterData>> m_rgrgParams = new List<List<ParameterData>>();
        int m_nMax;
 
        public ImageCache(int nMax)
        {
            m_nMax = nMax;
        }
 
        public int Count
        {
            get { return m_rgImages.Count; }
        }
 
        public void Clear()
        {
            m_rgImages.Clear();
            m_rgrgParams.Clear();
        }
 
        public bool Add(RawImage img, SimpleDatum sd, params ParameterData[] rgParams)
        {
            m_rgImages.Add(img);
 
            List<ParameterData> rgParam = new List<db.image.ParameterData>();
 
            if (sd is Datum)
            {
                Datum d = sd as Datum;
                string strVal = d.Tag as string;
                string strTag = d.TagName as string;
 
                if (!String.IsNullOrEmpty(strTag) && !String.IsNullOrEmpty(strVal))
                    rgParam.Add(new db.image.ParameterData(strTag, strVal, null, null, img.ID, false, img.SourceID.GetValueOrDefault()));
            }
 
            foreach (ParameterData param in rgParams)
            {
                param.ImageID = img.ID;
                param.SourceID = img.SourceID.GetValueOrDefault();
                rgParam.Add(param);
            }
 
            m_rgrgParams.Add(rgParam);
 
            if (m_rgImages.Count == m_nMax)
                return true;
 
            return false;
        }
 
        public List<RawImage> Images
        {
            get { return m_rgImages; }
        }
 
        public List<List<ParameterData>> Parameters
        {
            get { return m_rgrgParams; }
        }
    }
 
    public class LabelBoostDescriptor
    {
        int m_nLabel;
        double m_dfBoost;
 
        public LabelBoostDescriptor(int nLabel, double dfBoost)
        {
            m_nLabel = nLabel;
            m_dfBoost = dfBoost;
        }
 
        public int Label
        {
            get { return m_nLabel; }
        }
 
        public double Boost
        {
            get { return m_dfBoost; }
        }
    }
}