DatasetEx2.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using MyCaffe.basecode;
using System.Diagnostics;
using MyCaffe.basecode.descriptors;
 
namespace MyCaffe.db.image
{
    public class DatasetEx2 : IDisposable
    {
        CryptoRandom m_random = null;
        object m_syncObj = new object();
        DatasetFactory m_factory = null;
        DatasetDescriptor m_ds = null;
        ImageSet2 m_TestingImages = null;
        ImageSet2 m_TrainingImages = null;
        bool m_bUseTrainingImagesForTesting = false;
        List<Guid> m_rgUsers = new List<Guid>();
        int m_nOriginalDsId = 0;
        QueryStateCollection m_queryStates = new QueryStateCollection();
        long m_lDefaultQueryState = 0;
        ManualResetEvent m_evtRefreshScheduleCancel = new ManualResetEvent(false);
        ManualResetEvent m_evtRefreshScheduleRunning = new ManualResetEvent(false);
        ManualResetEvent m_evtRefreshScheduleDone = new ManualResetEvent(false);
        Thread m_refreshScheduleThread = null;
        int m_nRefreshUpdatePeriod = 0;
        double m_dfRegreshReplacementPct = 0;
        int m_nTrainingRefreshCount = 0;
        int m_nTestingRefreshCount = 0;
        bool m_bRefreshTraining = false;
        bool m_bRefreshTesting = false;
        Log m_log = null;
 
 
        public event EventHandler<CalculateImageMeanArgs> OnCalculateImageMean;
 
        public DatasetEx2(Guid user, DatasetFactory factory, CryptoRandom random)
        {
            m_random = random;
 
            if (user != Guid.Empty)
                m_rgUsers.Add(user);
 
            m_factory = new DatasetFactory(factory);
        }
 
 
        public int AddUser(Guid user)
        {
            m_rgUsers.Add(user);
            return m_rgUsers.Count;
        }
 
        public int RemoveUser(Guid user)
        {
            m_rgUsers.Remove(user);
            return m_rgUsers.Count;
        }
 
        public long Initialize(DatasetDescriptor ds, WaitHandle[] rgAbort, int nPadW = 0, int nPadH = 0, Log log = null, DB_LOAD_METHOD loadMethod = DB_LOAD_METHOD.LOAD_ALL, bool bSkipMeanCheck = false, int nImageDbLoadLimit = 0, int nImageDbAutoRefreshScheduledUpdateInMs = 0, double dfImageDbAutoRefreshScheduledReplacementPct = 0, bool bVerify = false)
        {
            lock (m_syncObj)
            {
                m_log = log;
 
                if (ds != null)
                    m_ds = ds;
 
                if (m_ds.TrainingSource.Width == -1 || m_ds.TrainingSource.Height == -1)
                {
                    log.WriteLine("WARNING: Cannot create a mean image for data sources that contain variable sized images.  The mean check will be skipped.", true);
                    bSkipMeanCheck = true;
                }
 
                bool bSilentLoad = (loadMethod == DB_LOAD_METHOD.LOAD_ON_DEMAND_BACKGROUND) ? true : false;
 
                m_TrainingImages = new ImageSet2(ImageSet2.TYPE.TRAIN, log, m_factory, m_ds.TrainingSource, loadMethod, m_random, rgAbort);
                m_TrainingImages.OnCalculateImageMean += OnCalculateImageMean;
                QueryState qsTraining = m_TrainingImages.Initialize(bSilentLoad, true, true, nImageDbLoadLimit, bVerify);
                SimpleDatum sdMean = null;
 
                if (!bSkipMeanCheck)
                {
                    bool bQueryOnly = false;
                    if (EntitiesConnection.GlobalDatabaseConnectInfo.Location != ConnectInfo.TYPE.LOCAL)
                        bQueryOnly = true;
 
                    sdMean = m_TrainingImages.GetImageMean(log, rgAbort, bQueryOnly);
                }
 
                if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
                    return 0;
 
                m_TestingImages = new ImageSet2(ImageSet2.TYPE.TEST, log, m_factory, m_ds.TestingSource, loadMethod, m_random, rgAbort);
                m_TestingImages.OnCalculateImageMean += OnCalculateImageMean;
                QueryState qsTesting = m_TestingImages.Initialize(bSilentLoad, true, true, nImageDbLoadLimit, bVerify);
 
                if (!bSkipMeanCheck)
                {
                    bool bSave = true;
                    if (EntitiesConnection.GlobalDatabaseConnectInfo.Location != ConnectInfo.TYPE.LOCAL)
                        bSave = false;
 
                    m_TestingImages.SetImageMean(sdMean, bSave);
                }
 
                if (EventWaitHandle.WaitAny(rgAbort, 0) != EventWaitHandle.WaitTimeout)
                    return 0;
 
                if (loadMethod == DB_LOAD_METHOD.LOAD_ALL && nImageDbLoadLimit > 0 && nImageDbAutoRefreshScheduledUpdateInMs > 0 && dfImageDbAutoRefreshScheduledReplacementPct > 0)
                    StartAutomaticRefreshSchedule(true, true, nImageDbAutoRefreshScheduledUpdateInMs, dfImageDbAutoRefreshScheduledReplacementPct);
 
                m_lDefaultQueryState = m_queryStates.CreateNewState(qsTraining, qsTesting);
                return m_lDefaultQueryState;
            }
        }
 
        public bool WaitForLoadingToComplete(bool bTraining, bool bTesting, int nWait = int.MaxValue)
        {
            if (bTraining)
            {
                if (!m_TrainingImages.WaitForLoadingToComplete(nWait))
                    return false;
            }
 
            if (bTesting)
            {
                if (!m_TestingImages.WaitForLoadingToComplete(nWait))
                    return false;
            }
 
            return true;
        }
 
        public bool WaitForRefreshToComplete(bool bTraining, bool bTesting, int nWait = int.MaxValue)
        {
            if (bTraining)
            {
                if (!m_TrainingImages.WaitForRefreshToComplete(nWait))
                    return false;
            }
 
            if (bTesting)
            {
                if (!m_TestingImages.WaitForRefreshToComplete(nWait))
                    return false;
            }
 
            return true;
        }
 
        public bool IsRefreshRunning(bool bTraining, bool bTesting)
        {
            bool bRunning = false;
 
            if (bTraining)
            {
                if (m_TrainingImages.IsRefreshRunning)
                    bRunning = true;
            }
 
            if (bTesting)
            {
                if (m_TestingImages.IsRefreshRunning)
                    bRunning = true;
            }
 
            return bRunning;
        }
 
        public void StartRefresh(bool bTraining = true, bool bTesting = true, double dfReplacementPct = 0.25)
        {
            if (bTraining)
                m_TrainingImages.StartRefresh(dfReplacementPct);
 
            if (bTesting)
                m_TestingImages.StartRefresh(dfReplacementPct);
        }
 
        public void StopRefresh(bool bTraining = true, bool bTesting = true)
        {
            if (bTraining)
                m_TrainingImages.StopRefresh();
 
            if (bTesting)
                m_TestingImages.StopRefresh();
        }
 
        public bool StartAutomaticRefreshSchedule(bool bTraining, bool bTesting, int nPeriodInMs, double dfReplacementPct)
        {
            if (m_refreshScheduleThread != null)
                return false;
 
            if (nPeriodInMs == 0 || dfReplacementPct == 0)
                return false;
 
            if (bTraining && m_TrainingImages.LoadMethod != DB_LOAD_METHOD.LOAD_ALL)
            {
                m_log.WriteLine("WARNING: The shared training data source '" + m_TrainingImages.Source.Name + "' appears to already be open with the '" + m_TrainingImages.LoadMethod.ToString() + "', so the load limit with automatic refresh will not be used.");
                return false;
            }
 
            if (bTesting && m_TestingImages.LoadMethod != DB_LOAD_METHOD.LOAD_ALL)
            {
                m_log.WriteLine("WARNING: The shared testing data source '" + m_TestingImages.Source.Name + "' appears to already be open with the '" + m_TestingImages.LoadMethod.ToString() + "', so the load limit with automatic refresh will not be used.");
                return false;
            }
 
            m_nRefreshUpdatePeriod = nPeriodInMs;
            m_dfRegreshReplacementPct = dfReplacementPct;
            m_bRefreshTraining = bTraining;
            m_bRefreshTesting = bTesting;
            m_refreshScheduleThread = new Thread(new ParameterizedThreadStart(refreshSchedule));
            m_evtRefreshScheduleCancel.Reset();
            m_evtRefreshScheduleDone.Reset();
            m_evtRefreshScheduleRunning.Reset();
            m_refreshScheduleThread.Start(new Tuple<int, double, bool, bool>(nPeriodInMs, dfReplacementPct, bTraining, bTesting));
            return m_evtRefreshScheduleRunning.WaitOne(1000);
        }
 
        public bool StopAutomaticRefreshSchedule(bool bTraining, bool bTesting)
        {
            if (m_refreshScheduleThread == null)
                return false;
 
            if (m_evtRefreshScheduleRunning.WaitOne(0))
            {
                m_evtRefreshScheduleCancel.Set();
                m_evtRefreshScheduleDone.WaitOne();
            }
 
            m_refreshScheduleThread = null;
 
            return true;
        }
 
        public bool GetAutomaticRefreshSchedule(out int nPeriodInMs, out double dfReplacementPct, out int nTrainingRefreshCount, out int nTestingRefreshCount)
        {
            nPeriodInMs = 0;
            dfReplacementPct = 0;
            nTrainingRefreshCount = m_nTrainingRefreshCount;
            nTestingRefreshCount = m_nTestingRefreshCount;
 
            if (m_refreshScheduleThread == null)
                return false;
 
            if (!m_evtRefreshScheduleRunning.WaitOne(0))
                return false;
 
            nPeriodInMs = m_nRefreshUpdatePeriod;
            dfReplacementPct = m_dfRegreshReplacementPct;
 
            return true;
        }
 
        private void refreshSchedule(object obj)
        {
            m_evtRefreshScheduleRunning.Set();
            m_nTrainingRefreshCount = 0;
            m_nTestingRefreshCount = 0;
 
            try
            {
                Tuple<int, double, bool, bool> arg = obj as Tuple<int, double, bool, bool>;
                int nPeriodMs = arg.Item1;
                double dfReplacementPct = arg.Item2;
                bool bTraining = arg.Item3;
                bool bTesting = arg.Item4;
                Stopwatch sw = new Stopwatch();
 
                sw.Start();
 
                m_log.WriteLine("INFO: Starting refresh cycle on '" + m_ds.Name + "' and running every " + m_nRefreshUpdatePeriod.ToString("N0") + " ms. with replacement percent = " + m_dfRegreshReplacementPct.ToString("P"));
 
                while (!m_evtRefreshScheduleCancel.WaitOne(1000))
                {
                    if (sw.ElapsedMilliseconds > m_nRefreshUpdatePeriod)
                    {
                        if (m_bRefreshTraining)
                        {
                            if (!m_TrainingImages.IsRefreshRunning)
                            {
                                m_log.WriteLine("INFO: Refreshing '" + m_TrainingImages.Source.Name + "'...");
                                m_TrainingImages.StartRefresh(m_dfRegreshReplacementPct);
                                m_nTrainingRefreshCount++;
                            }
                            else
                            {
                                m_log.WriteLine("'" + m_TrainingImages.Source.Name + "' refresh already running.");
                            }
                        }
 
                        if (m_bRefreshTesting)
                        {
                            if (!m_TestingImages.IsRefreshRunning)
                            {
                                m_log.WriteLine("INFO: Refreshing '" + m_TestingImages.Source.Name + "'...");
                                m_TestingImages.StartRefresh(m_dfRegreshReplacementPct);
                                m_nTestingRefreshCount++;
                            }
                            else
                            {
                                m_log.WriteLine("'" + m_TestingImages.Source.Name + "' refresh already running.");
                            }
                        }
 
                        sw.Restart();
                    }
                }
 
                if (m_TrainingImages.IsRefreshRunning)
                {
                    m_TrainingImages.StopRefresh();
                    m_log.WriteLine("Stopped '" + m_TrainingImages.Source.Name + "' refresh.");
                }
 
                if (m_TestingImages.IsRefreshRunning)
                {
                    m_TestingImages.StopRefresh();
                    m_log.WriteLine("Stopped '" + m_TestingImages.Source.Name + "' refresh.");
                }
            }
            finally
            {
                m_evtRefreshScheduleDone.Set();
                m_evtRefreshScheduleRunning.Reset();
            }
        }
 
 
        public long DefaultQueryState
        {
            get { return m_lDefaultQueryState; }
        }
 
        protected virtual void Dispose(bool bDisposing)
        {
            m_ds = null;
 
            StopAutomaticRefreshSchedule(true, true);
 
            if (m_TestingImages != null)
            {
                m_TestingImages.Dispose();
                m_TestingImages = null;
            }
 
            if (m_TrainingImages != null)
            {
                m_TrainingImages.Dispose();
                m_TrainingImages = null;
            }
 
            if (m_factory != null)
            {
                m_factory.Dispose();
                m_factory = null;
            }
        }
 
        public void Dispose()
        {
            Dispose(true);
        }
 
        public long CreateQueryState(bool bUseUniqueLabelIndexes = true, bool bUseUniqueImageIndexes = true, IMGDB_SORT sort = IMGDB_SORT.NONE)
        {
            QueryState qsTraining = m_TrainingImages.CreateQueryState(bUseUniqueLabelIndexes, bUseUniqueImageIndexes, sort);
            QueryState qsTesting = m_TestingImages.CreateQueryState(bUseUniqueLabelIndexes, bUseUniqueImageIndexes, sort);
            return m_queryStates.CreateNewState(qsTraining, qsTesting);
        }
 
        public bool SetDefaultQueryState(long lQueryState)
        {
            m_lDefaultQueryState = lQueryState;
            return true;
        }
 
        public bool FreeQueryState(long lHandle)
        {
            // Cannot free the default query state.
            if (lHandle == m_lDefaultQueryState)
                return false;
 
            return m_queryStates.FreeQueryState(lHandle);
        }
 
        public QueryState FindQueryState(long lQueryState, ImageSet2.TYPE type)
        {
            if (lQueryState == 0)
                lQueryState = m_lDefaultQueryState;
 
            if (type == ImageSet2.TYPE.TEST)
                return m_queryStates.GetTestingState(lQueryState);
            else
                return m_queryStates.GetTrainingState(lQueryState);
        }
 
        public void ReloadIndexing()
        {
            List<DbItem> rgItems = m_TrainingImages.ReloadIndexing();
            m_queryStates.ReIndexTraining(rgItems);
 
            rgItems = m_TrainingImages.ReloadIndexing();
            m_queryStates.ReIndexTesting(rgItems);
        }
 
        public void Relabel(LabelMappingCollection col)
        {
            List<DbItem> rgItems = m_TrainingImages.Relabel(col);
            m_queryStates.ReIndexTraining(rgItems);
 
            rgItems = m_TrainingImages.Relabel(col);
            m_queryStates.ReIndexTesting(rgItems);
        }
 
        public void ResetLabels()
        {
            List<DbItem> rgItems = m_TrainingImages.ResetLabels();
            m_queryStates.ReIndexTraining(rgItems);
 
            rgItems = m_TrainingImages.ResetLabels();
            m_queryStates.ReIndexTesting(rgItems);
        }
 
        public void ResetAllBoosts()
        {
            List<DbItem> rgItems = m_TrainingImages.ResetAllBoosts();
            m_queryStates.ReIndexTraining(rgItems);
 
            rgItems = m_TrainingImages.ResetAllBoosts();
            m_queryStates.ReIndexTesting(rgItems);
        }
 
        public bool UseTrainingImagesForTesting
        {
            get { return m_bUseTrainingImagesForTesting; }
            set { m_bUseTrainingImagesForTesting = value; }
        }
 
        public bool SaveImageMean(int nSrcId, SimpleDatum sd, bool bUpdate)
        {
            if (m_TestingImages.Source.ID != nSrcId &&
                m_TrainingImages.Source.ID != nSrcId)
                return false;
 
            return m_factory.SaveImageMean(sd, bUpdate, nSrcId);
        }
 
        public SimpleDatum QueryImageMean(int nSrcId)
        {
            if (m_TestingImages.Source.ID != nSrcId &&
                m_TrainingImages.Source.ID != nSrcId)
                return null;
 
            return m_factory.QueryImageMean(nSrcId);
        }
 
        public void Unload(bool bReload)
        {
            lock (m_syncObj)
            {
                m_TestingImages.Unload(bReload);
                m_TrainingImages.Unload(bReload);
            }
        }
 
        public double GetPercentageLoaded(out double dfTraining, out double dfTesting)
        {
            int nTrainingTotal = m_TrainingImages.GetTotalCount();
            int nTrainingLoaded = m_TrainingImages.GetLoadedCount();
            int nTestingTotal = m_TestingImages.GetTotalCount();
            int nTestingLoaded = m_TestingImages.GetLoadedCount();
 
            dfTraining = (double)nTrainingLoaded / (double)nTrainingTotal;
            dfTesting = (double)nTestingLoaded / (double)nTestingTotal;
 
            int nTotalLoaded = nTrainingLoaded + nTestingLoaded;
            int nTotalImages = nTrainingTotal + nTestingTotal;
 
            return (double)nTotalLoaded / (double)nTotalImages;
        }
 
        public ImageSet2 Find(int nSourceID)
        {
            if (m_TestingImages.Source.ID == nSourceID)
            {
                if (m_bUseTrainingImagesForTesting)
                    return m_TrainingImages;
 
                return m_TestingImages;
            }
 
            if (m_TrainingImages.Source.ID == nSourceID)
            {
                return m_TrainingImages;
            }
 
            return null;
        }
 
        public ImageSet2 Find(string strSource)
        {
            if (m_TestingImages.Source.Name == strSource)
            {
                if (m_bUseTrainingImagesForTesting)
                    return m_TrainingImages;
 
                return m_TestingImages;
            }
 
            if (m_TrainingImages.Source.Name == strSource)
            {
                return m_TrainingImages;
            }
 
            return null;
        }
 
        public DatasetDescriptor Descriptor
        {
            get { return m_ds; }
        }
 
        public int DatasetID
        {
            get { return (m_ds == null) ? 0 : m_ds.ID; }
            set { m_ds.ID = value; }
        }
 
        public int OriginalDatasetID
        {
            get { return m_nOriginalDsId; }
        }
 
        public string DatasetName
        {
            get { return m_ds.Name; }
        }
    }
}