Net.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.IO;
using System.Diagnostics;
using MyCaffe.basecode;
using MyCaffe.db.image;
using MyCaffe.param;
using MyCaffe.data;
using MyCaffe.layers;
using MyCaffe.fillers;
 
namespace MyCaffe.common
{
    public class Net<T> : IDisposable 
    {       
        NetParameter m_param;
        CudaDnn<T> m_cuda;
        Log m_log;
 
        // The network name.
        string m_strName;
 
        // The phase: TRAIN or TEST
        Phase m_phase = Phase.NONE;
        Phase m_phaseOriginal = Phase.NONE;
 
        // Individual layers in the net.
        List<Layer<T>> m_rgLayers = new List<Layer<T>>();
        List<string> m_rgstrLayerNames = new List<string>();
        DictionaryEx<string, int> m_rgLayerNamesIndex = new DictionaryEx<string, int>(0);
        List<bool> m_rgbLayerNeedBackward = new List<bool>();
 
        // The blobs storing intermediate results between the layer.
        BlobCollection<T> m_colBlobs = new BlobCollection<T>();
        List<string> m_rgstrBlobNames = new List<string>();
        DictionaryEx<string, int> m_rgBlobNamesIndex = new DictionaryEx<string, int>(0);
        List<bool> m_rgbBlobNeedBackward = new List<bool>();
 
        // The bottom vecs storing the vectors containing the input for each layer.
        // They don't actually host the blobs (blobs_ does), so we simply store
        // the reference.
        List<BlobCollection<T>> m_rgcolBottomVecs = new List<BlobCollection<T>>();
        List<List<int>> m_rgrgnBottomIdVecs;
        List<List<bool>> m_rgrgbBottomNeedBackward = new List<List<bool>>();
        
        // The top vecs stores the vecotrs containing the output of each layer.
        List<BlobCollection<T>> m_rgcolTopVecs = new List<BlobCollection<T>>();
        List<List<int>> m_rgrgnTopIdVecs = new List<List<int>>();
 
        // Vector of weight in the loss (or objective) function of each net blob,
        // indexed by blob_id.
        List<double> m_rgdfBlobLossWeights = new List<double>();
        List<List<int>> m_rgrgnParamIdVecs = new List<List<int>>();
        List<int> m_rgnParamOwners = new List<int>();
        List<string> m_rgstrParamDisplayNames = new List<string>();
        List<KeyValuePair<int, int>> m_rgParamLayerIndices = new List<KeyValuePair<int, int>>();
        DictionaryEx<string, int> m_rgParamNamesIndex = new DictionaryEx<string, int>(0);
 
        // blob indices for the input and the output of the net.
        List<int> m_rgnNetInputBlobIndices = new List<int>();
        List<int> m_rgnNetOutputBlobIndices = new List<int>();
        BlobCollection<T> m_colNetInputBlobs = new BlobCollection<T>();
        BlobCollection<T> m_colNetOutputBlobs = new BlobCollection<T>();
 
        // The parameters in the network.
        BlobCollection<T> m_colParams = new BlobCollection<T>();
        BlobCollection<T> m_colLearnableParams = new BlobCollection<T>();
 
        // The mapping from params -> learnable_params : we have
        // learnable_param_ids.Count == params.Count,
        // and learnable_params[learnable_param_ids[i]] == params[i]
        // if and only if params[i] is an 'owner'; otherwise params[i] is a sharer
        // and learnable_params[learnable_param_ids[i]] gives its owner.
        List<int> m_rgnLearnableParamIds = new List<int>();
        
        // The learning rate multipliers from learnable params.
        List<double?> m_rgdfParamsLr = new List<double?>();
 
        // The weight decay multipliers for learnable params.
        List<double?> m_rgdfParamsWeightDecay = new List<double?>();
 
        // The bytes of memory used by this net
        long m_lMemoryUsed = 0;
 
        // Whether to compute and display debug info for the net.
        bool m_bDebugInfo = false;
 
        // The in-memory database passed through to the data layer(s).
        IXDatabaseBase m_db = null;
 
        // Cancel event used to cancel training and testing.
        CancelEvent m_evtCancel;
 
        // Store the last forward input to the data layer, if any (currently only supported by BatchDataInput).
        BatchInput m_lastBatchInput = null;
 
        // When enabled, the best result mask zero's out all data items except for those that have the greatest value
        string m_strBestResultTargetNodeToMask = null;
        int m_nBestResultCount = 5;
        BEST_RESULT_TYPE m_nBestResultType = BEST_RESULT_TYPE.BY_CHANNEL;
 
        long m_hWorkspaceData = 0;  // shared among the layers, only grows in size.
        ulong m_lWorkspaceSizeInBytes = 0;
 
        Net<T> m_sharedNet = null;
        bool m_bBreakOnFirstNan = false;
        bool m_bDetectDetailedNans = false;
        bool m_bEnableLayerDebugging = false;
        Blob<T> m_blobWork = null;
        List<Layer<T>> m_rgConnectedLayers = new List<Layer<T>>();
        Blob<T> m_debugBlob = null;
        int m_nLastNonFrozenLayerIdx = 0;
        string m_strDataSource = null;
        Layer<T> m_labelMappingLayer = null;
        bool m_bFirstForwardInputWarning = true;
 
        public event EventHandler<WorkspaceArgs> OnGetWorkspace;
        public event EventHandler<WorkspaceArgs> OnSetWorkspace;
        public event EventHandler<GetIterationArgs> OnGetIteration;
 
#pragma warning disable 1591
 
        public enum BEST_RESULT_TYPE 
        {
            BY_CHANNEL,
            BY_WEIGHT
        }
 
#pragma warning restore 1591
 
        public Net(CudaDnn<T> cuda, Log log, NetParameter p, CancelEvent evtCancel, IXDatabaseBase db, Phase phaseOverride = Phase.NONE, AutoResetEvent evtTrainingCompleted = null, Net<T> sharedNet = null, onGetWorkspace getws = null, onSetWorkspace setws = null)
        {
            m_sharedNet = sharedNet;
            m_db = db;
            m_cuda = cuda;
            m_log = log;
            m_blobWork = new Blob<T>(cuda, log);
 
            m_evtCancel = evtCancel;
 
            if (getws != null)
                OnGetWorkspace += new EventHandler<WorkspaceArgs>(getws);
 
            if (setws != null)
                OnSetWorkspace += new EventHandler<WorkspaceArgs>(setws);
 
            Init(p, phaseOverride, evtTrainingCompleted);
        }
 
        protected virtual void Dispose(bool bDisposing)
        {
            foreach (Layer<T> layer in m_rgConnectedLayers)
            {
                if (layer is DataLayer<T>)
                    ((DataLayer<T>)layer).Disconnect();
            }
 
            m_rgConnectedLayers.Clear();
 
            foreach (Layer<T> layer in m_rgLayers)
            {
                layer.Dispose();
            }
 
            m_rgLayers.Clear();
 
            if (m_colBlobs != null)
            {
                m_colBlobs.Dispose();
                m_colBlobs = null;
            }
 
            foreach (BlobCollection<T> b in m_rgcolBottomVecs)
            {
                b.Dispose();
            }
 
            m_rgcolBottomVecs.Clear();
 
            foreach (BlobCollection<T> b in m_rgcolTopVecs)
            {
                b.Dispose();
            }
 
            m_rgcolTopVecs.Clear();
 
            if (m_colNetOutputBlobs != null)
            {
                m_colNetOutputBlobs.Dispose();
                m_colNetOutputBlobs = null;
            }
 
            if (m_colParams != null)
            {
                m_colParams.Dispose();
                m_colParams = null;
            }
 
            if (m_colLearnableParams != null)
            {
                m_colLearnableParams.Dispose();
                m_colLearnableParams = null;
            }
 
            if (m_hWorkspaceData != 0)
            {
                m_cuda.DisableGhostMemory();
                m_cuda.FreeMemory(m_hWorkspaceData);
                m_cuda.ResetGhostMemory();
                m_hWorkspaceData = 0;
                m_lWorkspaceSizeInBytes = 0;
            }
 
            if (m_blobWork != null)
            {
                m_blobWork.Dispose();
                m_blobWork = null;
            }
        }
 
        public void Dispose()
        {
            if (m_debugBlob != null)
            {
                m_debugBlob.Dispose();
                m_debugBlob = null;
            }
 
            Dispose(true);
        }
 
        private void scanForRecommendations(NetParameter p)
        {
            for (int i = 0; i < p.layer.Count; i++)
            {
                if (p.layer[i].type == LayerParameter.LayerType.LSTM_SIMPLE)
                    m_log.WriteLine("WARNING: Layer '" + p.layer[i].name + "' uses the LSTM_SIMPLE type, we recommend using the LSTM with CUDNN engine instead.");
 
                if (p.layer[i].type == LayerParameter.LayerType.LSTM && p.layer[i].recurrent_param.engine == EngineParameter.Engine.CAFFE)
                    m_log.WriteLine("WARNING: Layer '" + p.layer[i].name + "' uses the LSTM type with the CAFFE engine (which is quite slow), we recommend using the LSTM with CUDNN engine instead.");
            }
        }
 
        public void Init(NetParameter p, Phase phaseOverride = Phase.NONE, AutoResetEvent evtTrainingCompleted = null)
        {
            try
            {
                m_bFirstForwardInputWarning = true;
                m_param = p;
 
                // Set phase from the state.
                if (phaseOverride != Phase.NONE)
                {
                    m_phase = phaseOverride;
                    p.state.phase = m_phase;
                }
                else
                {
                    m_phase = p.state.phase;
                }
 
                m_phaseOriginal = m_phase;
 
                scanForRecommendations(p);
 
                // Filter layser based on their include/exclude rules and
                // the current NetState.
                NetParameter filtered_param = FilterNet(p);
                m_log.WriteLine("Initializing net from parameters: " + filtered_param.DebugString());
 
                // Create a copy of filtered_param with splits added where necessary;
                NetParameter param = InsertSplits(filtered_param);
 
                // Basically, build all the layers and set up their connections.
                m_strName = param.name;
 
                DictionaryEx<string, int> blob_name_to_idx = new DictionaryEx<string, int>(0);
                List<string> available_blobs = new List<string>();
 
                m_log.CHECK(param.input_dim.Count == 0 || param.input_shape.Count == 0, "Must specify either input_shape OR depreciated input_dim, not both.");
 
                if (param.input_dim.Count > 0)
                {
                    // Depreciated 4D dimensions.
                    m_log.CHECK_EQ(param.input.Count * 4, param.input_dim.Count, "Incorrect inpub blob dimension specification.");
                }
                else
                {
                    m_log.CHECK_EQ(param.input.Count, param.input_shape.Count, "Exactly one input_shape must be specified per input.");
                }
 
                m_lMemoryUsed = 0;
 
                // Set the input blobs
                for (int input_id = 0; input_id < param.input.Count; input_id++)
                {
                    int layer_id = -1; // inputs have fake layer ID = -1
                    AppendTop(param, layer_id, input_id, available_blobs, blob_name_to_idx);
                }
 
                // For each layer, set up its input and output
                m_rgcolBottomVecs = new List<BlobCollection<T>>();
                m_rgcolTopVecs = new List<BlobCollection<T>>();
                m_rgrgnBottomIdVecs = new List<List<int>>();
                m_rgrgnParamIdVecs = new List<List<int>>();
                m_rgrgnTopIdVecs = new List<List<int>>();
                m_rgrgbBottomNeedBackward = new List<List<bool>>();
 
                Dictionary<string, Layer<T>> rgSyncLayers = new Dictionary<string, Layer<T>>();
 
                for (int layer_id = 0; layer_id < param.layer.Count; layer_id++)
                {
                    m_rgcolBottomVecs.Add(new BlobCollection<T>());
                    m_rgcolTopVecs.Add(new BlobCollection<T>());
                    m_rgrgnBottomIdVecs.Add(new List<int>());
                    m_rgrgnTopIdVecs.Add(new List<int>());
                    m_rgrgnParamIdVecs.Add(new List<int>());
                    m_rgrgbBottomNeedBackward.Add(new List<bool>());
 
                    // Inherit phase from net if unset.
                    if (param.layer[layer_id].phase == Phase.NONE)
                        param.layer[layer_id].phase = m_phase;
 
                    // Setup layer.
                    LayerParameter layer_param = param.layer[layer_id];
                    if (layer_param.propagate_down.Count > 0)
                        m_log.CHECK_EQ(layer_param.propagate_down.Count, layer_param.bottom.Count, "propagate_down param must be specified either 0 or bottom.Count times.");
 
                    //-------------------------------------------
                    //  When sharing the blobs of another net
                    //  (e.g. The run net does this when also
                    //  training, to save memory)
                    //  pass the parameters and internal blobs
                    //  into the layer_parameter thus allowing
                    //  each layer to share blobs as appropriate.
                    //-------------------------------------------
                    LayerParameter layer_paramEx = layer_param;
                    if (m_sharedNet != null)
                    {
                        Layer<T> sharedLayer = m_sharedNet.FindLayer(layer_param.type, layer_param.name);
                        layer_paramEx = new LayerParameterEx<T>(layer_param, m_sharedNet.parameters, m_sharedNet.layer_blobs(layer_param.name), sharedLayer);
                    }
 
                    layer_paramEx.solver_count = m_param.solver_count;
                    layer_paramEx.solver_rank = m_param.solver_rank;
 
                    // Setup layer continued.
                    Layer<T> layer1 = Layer<T>.Create(m_cuda, m_log, layer_paramEx, m_evtCancel, m_db, new TransferInput(getInput, setInput));
                    layer1.OnGetWorkspace += layer_OnGetWorkspace;
                    layer1.OnSetWorkspace += layer_OnSetWorkspace;
                    layer1.OnGetIteration += layer_OnGetIteration;
 
                    if (layer1.type == LayerParameter.LayerType.DATA)
                        m_strDataSource = layer1.layer_param.data_param.source;
                    else if (layer1.type == LayerParameter.LayerType.LABELMAPPING)
                        m_labelMappingLayer = layer1;
 
                    m_rgLayers.Add(layer1);
 
                    m_rgstrLayerNames.Add(layer_param.name);
                    m_log.WriteLine("Creating layer " + layer_param.name);
 
                    bool need_backward = false;
 
                    // Figure out this layer's input and output
                    for (int bottom_id = 0; bottom_id < layer_param.bottom.Count; bottom_id++)
                    {
                        int blob_id = AppendBottom(param, layer_id, bottom_id, available_blobs, blob_name_to_idx);
 
                        // If a blob needs backward, this layer should provide it.
                        need_backward |= m_rgbBlobNeedBackward[blob_id];
                    }
 
                    int num_top = layer_param.top.Count;
                    for (int top_id = 0; top_id < num_top; top_id++)
                    {
                        // Ignore top's named 'null'
                        if (param.layer[layer_id] != null && param.layer[layer_id].top[top_id] == "null")
                            continue;
 
                        AppendTop(param, layer_id, top_id, available_blobs, blob_name_to_idx);
 
                        // Collect Input layer tops as Net inputs.
                        if (layer_param.type == LayerParameter.LayerType.INPUT)
                        {
                            int nBlobID = blobs.Count - 1;
                            m_rgnNetInputBlobIndices.Add(nBlobID);
                            m_colNetInputBlobs.Add(blobs[nBlobID]);
                        }
                    }
 
                    // If the layer specifies that AutoTopBlobs() == true and the LayerParameter
                    // specified fewer than the required number (as specified by
                    // ExactNumTopBlobs() or MinTopBlobs()), allocate them here.
                    Layer<T> layer = m_rgLayers[layer_id];
                    if (layer.AutoTopBlobs)
                    {
                        int needed_num_top = Math.Max(layer.MinTopBlobs, layer.ExactNumTopBlobs);
 
                        while (num_top < needed_num_top)
                        {
                            // Add 'anonymous' top blobs -- do not modify available_blobs or
                            // blob_name_to_idx as we don't want these blbos to be usable as input
                            // to other layers.
                            AppendTop(param, layer_id, num_top, null, null);
                            num_top++;
                        }
                    }
 
                    // After this layer is connected, set it up.
                    m_rgLayers[layer_id].SetNetParameterUsed(param); // used for label mapping
                    m_rgLayers[layer_id].Setup(m_rgcolBottomVecs[layer_id], m_rgcolTopVecs[layer_id]);
            
                    // Setup the layer.
                    m_log.WriteLine("Setting up " + m_rgstrLayerNames[layer_id]);
 
                    for (int top_id = 0; top_id < m_rgcolTopVecs[layer_id].Count; top_id++)
                    {
                        int nIdx = m_rgrgnTopIdVecs[layer_id][top_id];
 
                        if (m_rgdfBlobLossWeights.Count <= nIdx)
                            Utility.Resize<double>(ref m_rgdfBlobLossWeights, nIdx + 1, 0.0);
 
                        double dfLoss = layer.loss(top_id);
 
                        m_rgdfBlobLossWeights[nIdx] = dfLoss;
 
                        if (m_log.IsEnabled)
                        {
                            string strOut = "Top shape: " + m_rgcolTopVecs[layer_id][top_id].shape_string;
 
                            if (dfLoss != 0)
                                strOut += "  with loss weight " + dfLoss.ToString();
 
                            m_log.WriteLine(strOut);
                        }
 
                        m_lMemoryUsed += m_rgcolTopVecs[layer_id][top_id].count();
                    }
 
                    m_log.WriteLine("Memory required for data: " + (m_lMemoryUsed * Utility.BaseTypeSize<T>()).ToString());
 
                    int param_size = layer_param.GetParameterCount();
                    int num_param_blobs = m_rgLayers[layer_id].blobs.Count();
                    m_log.CHECK_LE(param_size, num_param_blobs, "Too many params specified for layer " + layer_param.name);
 
                    ParamSpec default_param_spec = new ParamSpec();
 
                    for (int param_id = 0; param_id < num_param_blobs; param_id++)
                    {
                        ParamSpec param_spec = (param_id < param_size) ? layer_param.parameters[param_id] : default_param_spec;
                        bool param_need_backward = (param_spec.lr_mult != 0.0) ? true : false;
 
                        need_backward |= param_need_backward;
                        m_rgLayers[layer_id].set_param_propagate_down(param_id, param_need_backward);
                    }
 
                    for (int param_id = 0; param_id < num_param_blobs; param_id++)
                    {
                        AppendParam(param, layer_id, param_id);
                    }
 
                    // Connect up the synced layers, if any.
                    if (layer_param.type == LayerParameter.LayerType.DATA)
                    {
                        if (layer_param.data_param.synchronize_with != null)
                        {
                            rgSyncLayers.Add(layer_param.data_param.synchronize_with, m_rgLayers[layer_id]);
                        }
                        else
                        {
                            List<KeyValuePair<string, Layer<T>>> rgSyncLayers1 = rgSyncLayers.ToList();
                            for (int i = 0; i < rgSyncLayers1.Count; i++)
                            {
                                if (rgSyncLayers1[i].Key == layer_param.name)
                                {
                                    ((DataLayer<T>)layer).Connect((DataLayer<T>)rgSyncLayers1[i].Value);
                                    m_rgConnectedLayers.Add(layer);
                                    rgSyncLayers.Remove(layer_param.name);
                                }
                            }
                        }
                    }
 
                    // Finally, set the backward flag.
                    m_rgbLayerNeedBackward.Add(need_backward);
 
                    if (need_backward)
                    {
                        for (int top_id = 0; top_id < m_rgrgnTopIdVecs[layer_id].Count; top_id++)
                        {
                            int nIdx = m_rgrgnTopIdVecs[layer_id][top_id];
                            m_rgbBlobNeedBackward[nIdx] = true;
                        }
                    }
                }
 
                if (rgSyncLayers.Count > 0)
                {
                    string strLayer = "";
 
                    foreach (KeyValuePair<string, Layer<T>> kv in rgSyncLayers)
                    {
                        strLayer += kv.Key + ", ";
                    }
 
                    strLayer = strLayer.TrimEnd(',', ' ');
                    m_log.FAIL("The following layers are expected to be marked with 'synchronize_target = True': '" + strLayer + "'.");
                }
 
                // Go through the net backwards to determine which blobs contribute to the
                // loss.  We can skip backward computation for blobs that don't contribute
                // to the loss.
                // Also checks if all bottom blobs don't need backward computation (possible
                // because the skip_propagate_down param) and so we can skip backward
                // computation for the entire layer.
                List<string> blobs_under_loss = new List<string>();
                List<string> blobs_skip_backp = new List<string>();
 
                for (int layer_id = m_rgLayers.Count - 1; layer_id >= 0; layer_id--)
                {
                    bool layer_contributes_loss = false;
                    bool layer_skip_propagate_down = true;
 
                    for (int top_id = 0; top_id < m_rgcolTopVecs[layer_id].Count; top_id++)
                    {
                        int nIdx = m_rgrgnTopIdVecs[layer_id][top_id];
                        string blob_name = m_rgstrBlobNames[nIdx];
 
                        if (m_rgLayers[layer_id].loss(top_id) != 0 || blobs_under_loss.Contains(blob_name))
                            layer_contributes_loss = true;
 
                        if (!blobs_skip_backp.Contains(blob_name))
                            layer_skip_propagate_down = false;
 
                        if (layer_contributes_loss && !layer_skip_propagate_down)
                            break;
                    }
 
                    // If this layer can skip backward computation, also all of its bottom blobs
                    // don't need backpropagation
                    if (m_rgbLayerNeedBackward[layer_id] && layer_skip_propagate_down)
                    {
                        m_rgbLayerNeedBackward[layer_id] = false;
 
                        for (int bottom_id = 0; bottom_id < m_rgcolBottomVecs[layer_id].Count; bottom_id++)
                        {
                            m_rgrgbBottomNeedBackward[layer_id][bottom_id] = false;
                        }
                    }
 
                    if (!layer_contributes_loss)
                        m_rgbLayerNeedBackward[layer_id] = false;
 
                    if (m_log.IsEnabled)
                    {
                        if (m_rgbLayerNeedBackward[layer_id])
                            m_log.WriteLine(m_rgstrLayerNames[layer_id] + " needs backward computation.");
                        else
                            m_log.WriteLine(m_rgstrLayerNames[layer_id] + " does not need backward computation.");
                    }
 
                    for (int bottom_id = 0; bottom_id < m_rgcolBottomVecs[layer_id].Count; bottom_id++)
                    {
                        if (layer_contributes_loss)
                        {
                            int nIdx = m_rgrgnBottomIdVecs[layer_id][bottom_id];
                            string blob_name = m_rgstrBlobNames[nIdx];
 
                            blobs_under_loss.Add(blob_name);
                        }
                        else
                        {
                            m_rgrgbBottomNeedBackward[layer_id][bottom_id] = false;
                        }
 
                        if (!m_rgrgbBottomNeedBackward[layer_id][bottom_id])
                        {
                            int nIdx = m_rgrgnBottomIdVecs[layer_id][bottom_id];
                            string blob_name = m_rgstrBlobNames[nIdx];
 
                            blobs_skip_backp.Add(blob_name);
                        }
                    }
                }
 
                // Handle force_backward if needed.
                if (param.force_backward)
                {
                    for (int layer_id = 0; layer_id < m_rgLayers.Count; layer_id++)
                    {
                        m_rgbLayerNeedBackward[layer_id] = true;
 
                        for (int bottom_id = 0; bottom_id < m_rgrgbBottomNeedBackward[layer_id].Count; bottom_id++)
                        {
                            m_rgrgbBottomNeedBackward[layer_id][bottom_id] = m_rgrgbBottomNeedBackward[layer_id][bottom_id] || m_rgLayers[layer_id].AllowForceBackward(bottom_id);
 
                            int nIdx = m_rgrgnBottomIdVecs[layer_id][bottom_id];
                            m_rgbBlobNeedBackward[nIdx] = m_rgbBlobNeedBackward[nIdx] || m_rgrgbBottomNeedBackward[layer_id][bottom_id];
                        }
 
                        for (int param_id = 0; param_id < m_rgLayers[layer_id].blobs.Count; param_id++)
                        {
                            m_rgLayers[layer_id].set_param_propagate_down(param_id, true);
                        }
                    }
                }
 
                // In the end, all remaining blobs are considered output blobs.
                foreach (string blob_name in available_blobs)
                {
                    m_log.WriteLine("This network produces output " + blob_name);
                    int nIdx = blob_name_to_idx[blob_name];
                    Blob<T> blob = m_colBlobs[nIdx];
 
                    m_colNetOutputBlobs.Add(blob);
                    m_rgnNetOutputBlobIndices.Add(nIdx);
                }
 
                for (int blob_id = 0; blob_id < m_rgstrBlobNames.Count; blob_id++)
                {
                    string blob_name = m_rgstrBlobNames[blob_id];
                    m_rgBlobNamesIndex[blob_name] = blob_id;
                }
 
                for (int layer_id = 0; layer_id < m_rgstrLayerNames.Count; layer_id++)
                {
                    string layer_name = m_rgstrLayerNames[layer_id];
                    m_rgLayerNamesIndex[layer_name] = layer_id;
                }
 
                if (m_sharedNet == null)
                    ShareWeights();
 
                // Set last non frozen layer to optimize back propagation
                // by only performing back-brpopagation up to the last
                // non-frozen layer.
                for (int i = 0; i < m_rgLayers.Count; i++)
                {
                    m_nLastNonFrozenLayerIdx = i;
 
                    if (!m_rgLayers[i].layer_param.freeze_learning && m_rgLayers[i].layer_param.type != LayerParameter.LayerType.SPLIT)
                        break;
                }
 
                LossLayer<T> lossLayer = null;
                for (int i=m_rgLayers.Count-1; i>=0; i--)
                {
                    if (m_rgLayers[i] is LossLayer<T>)
                    {
                        lossLayer = m_rgLayers[i] as LossLayer<T>;
                        break;
                    }
                }
 
                // Connect any loss events
                if (lossLayer != null)
                {
                    for (int i = 0; i < m_rgLayers.Count; i++)
                    {
                        if (m_rgLayers[i].layer_param.connect_loss_event)
                            m_rgLayers[i].ConnectLoss(lossLayer);
                    }
                }
 
                m_bDebugInfo = param.debug_info;
                m_log.WriteLine("Network initialization done.");
            }
            catch (Exception excpt)
            {
                foreach (Layer<T> layer in m_rgConnectedLayers)
                {
                    ((DataLayer<T>)layer).Disconnect();
                }
 
                m_rgConnectedLayers.Clear();
 
                foreach (Layer<T> layer in m_rgLayers)
                {
                    layer.Dispose();
                }
 
                m_rgLayers.Clear();
                m_rgstrLayerNames.Clear();
                m_rgLayerNamesIndex.Clear();
                m_rgbBlobNeedBackward.Clear();
                m_colBlobs.Dispose();
                m_rgBlobNamesIndex.Clear();
                m_rgbBlobNeedBackward.Clear();
                m_rgcolBottomVecs.Clear();
                m_rgrgbBottomNeedBackward.Clear();
                m_rgcolTopVecs.Clear();
                m_rgrgnTopIdVecs.Clear();
                m_rgdfBlobLossWeights.Clear();
                m_rgrgnParamIdVecs.Clear();
                m_rgnParamOwners.Clear();
                m_rgstrParamDisplayNames.Clear();
                m_rgParamLayerIndices.Clear();
                m_rgParamNamesIndex.Clear();
                m_rgnNetInputBlobIndices.Clear();
                m_rgnNetOutputBlobIndices.Clear();
                m_colNetInputBlobs.Clear();
                m_colNetOutputBlobs.Clear();
                m_colParams.Clear();
                m_colLearnableParams.Clear();
                m_rgnLearnableParamIds.Clear();
                m_rgdfParamsLr.Clear();
                m_rgdfParamsWeightDecay.Clear();
                m_lMemoryUsed = 0;
                m_bDebugInfo = false;
                m_db = null;
                throw excpt;
            }
        }
 
        private void layer_OnDebug(object sender, GetWorkBlobArgs<T> e)
        {
            e.Blob = m_blobWork;
        }
 
        private void layer_OnGetIteration(object sender, GetIterationArgs e)
        {
            if (OnGetIteration != null)
                OnGetIteration(sender, e);
        }
 
        private void layer_OnSetWorkspace(object sender, WorkspaceArgs e)
        {
            if (e.WorkspaceSizeInBytes == 0)
                return; 
 
            if (OnSetWorkspace != null)
            {
                OnSetWorkspace(sender, e);
                return;
            }
 
            m_cuda.DisableGhostMemory();
 
            if (e.WorkspaceSizeInBytes > m_lWorkspaceSizeInBytes)
            {
                m_lWorkspaceSizeInBytes = e.WorkspaceSizeInBytes;
 
                if (m_hWorkspaceData != 0)
                    m_cuda.FreeMemory(m_hWorkspaceData);
 
                ulong lCount = CudaDnn<T>.ConvertByteSizeToCount(m_lWorkspaceSizeInBytes);
                m_hWorkspaceData = m_cuda.AllocMemory((long)lCount);
            }
 
            m_cuda.ResetGhostMemory();
        }
 
        private void layer_OnGetWorkspace(object sender, WorkspaceArgs e)
        {
            if (OnGetWorkspace != null)
            {
                OnGetWorkspace(sender, e);
                return;
            }
 
            e.WorkspaceData = m_hWorkspaceData;
            e.WorkspaceSizeInBytes = m_lWorkspaceSizeInBytes;
        }
 
        public Phase SetPhase(Phase phase)
        {
            Phase phaseOriginal = m_phase;
 
            m_phase = phase;
 
            for (int i = 0; i < m_rgLayers.Count; i++)
            {
                m_rgLayers[i].SetPhase(phase);
            }
 
            return phaseOriginal;
        }
 
        public void RestorePhase()
        {
            m_phase = m_phaseOriginal;
 
            for (int i = 0; i < m_rgLayers.Count; i++)
            {
                m_rgLayers[i].SetPhase(m_phase);
            }
        }
 
        public bool EnableBreakOnFirstNaN
        {
            get { return m_bBreakOnFirstNan; }
            set { m_bBreakOnFirstNan = value; }
        }
 
        public bool EnableDetailedNanDetection
        {
            get { return m_bDetectDetailedNans; }
            set { m_bDetectDetailedNans = value; }
        }
 
        public bool EnableLayerDebugging
        {
            get { return m_bEnableLayerDebugging; }
            set
            {
                if (m_bEnableLayerDebugging == value)
                    return;
 
                foreach (Layer<T> layer in m_rgLayers)
                {
                    // Enable layer debugging which checks for NAN/INF on each fwd/bwd pass, but is much
                    // slower and therefore only for debugging.
                    if (value)
                        layer.SetOnDebug(layer_OnDebug);
                    else
                        layer.ResetOnDebug(layer_OnDebug);
                }
 
                m_bEnableLayerDebugging = value;
 
                if (m_bEnableLayerDebugging)
                    m_log.WriteLine("WARNING: Layer debugging enabled, training will be slow.");
                else
                    m_log.WriteLine("Layer debugging disabled.");
            }
        }
 
#pragma warning disable 1591
 
        public void EnableBestResultMask(string strTargetNode, int nBestResultCount = 5, BEST_RESULT_TYPE resultType = BEST_RESULT_TYPE.BY_CHANNEL) 
        {
            m_strBestResultTargetNodeToMask = strTargetNode;
            m_nBestResultCount = nBestResultCount;
            m_nBestResultType = resultType;
        }
 
        public void DisableBestResultMask() 
        {
            m_strBestResultTargetNodeToMask = null;
            m_nBestResultCount = 50;
        }
 
#pragma warning restore 1591
 
        public string ActiveLabelCounts
        {
            get
            {
                string strSrc = m_strDataSource; // Set during Init
 
                if (m_labelMappingLayer != null)
                    return ((LabelMappingLayer<T>)m_labelMappingLayer).GetActualLabelCounts(strSrc);
 
                if (string.IsNullOrEmpty(strSrc))
                    return "n/a";
 
                if (m_db.GetVersion() == DB_VERSION.TEMPORAL)
                    return "n/a";
 
                return ((IXImageDatabaseBase)m_db).GetLabelCountsAsTextFromSourceName(strSrc);
            }
        }
 
        public string LabelQueryHitPercents
        {
            get
            {
                string strSrc = m_strDataSource; // Set during Init
 
                if (string.IsNullOrEmpty(strSrc))
                    return "n/a";
 
                if (m_db.GetVersion() == DB_VERSION.TEMPORAL)
                    return "n/a";
 
                return ((IXImageDatabaseBase)m_db).GetLabelQueryHitPercentsAsTextFromSourceName(strSrc);
            }
        }
 
        public string LabelQueryEpochs
        {
            get
            {
                string strSrc = m_strDataSource; // Set during Init
 
                if (string.IsNullOrEmpty(strSrc))
                    return "n/a";
 
                if (m_db.GetVersion() == DB_VERSION.TEMPORAL)
                    return "n/a";
 
                return ((IXImageDatabaseBase)m_db).GetLabelQueryEpocsAsTextFromSourceName(strSrc);
            }
        }
 
        public string BoostQueryHitPercents
        {
            get
            {
                string strSrc = m_strDataSource; // Set during Init
 
                if (string.IsNullOrEmpty(strSrc))
                    return "n/a";
 
                if (m_db.GetVersion() == DB_VERSION.TEMPORAL)
                    return "n/a";
 
                return ((IXImageDatabaseBase)m_db).GetBoostQueryHitPercentsAsTextFromSourceName(strSrc);
            }
        }
 
        public void SetEnablePassthrough(bool bEnable)
        {
            foreach (Layer<T> layer in m_rgLayers)
            {
                layer.SetEnablePassthrough(bEnable);
            }   
        }
 
        private BatchInput getInput()
        {
            return m_lastBatchInput;
        }
 
        private void setInput(BatchInput biInput)
        {
            m_lastBatchInput = biInput;
        }
 
        public NetParameter FilterNet(NetParameter param)
        {
            NetState net_state = param.state;
            NetParameter param_filtered = param.Clone(false);
 
            for (int i = 0; i < param.layer.Count; i++)
            {
                LayerParameter layer_param = param.layer[i];
                string layer_name = layer_param.name;
 
                m_log.CHECK(layer_param.include.Count == 0 || layer_param.exclude.Count == 0, "Specify either include rules or exclude rules; not both.");
 
                // If no include rules are specified, the layer is included by default and
                // only excluded if it meets one of the exclude rules.
                bool layer_included = (layer_param.include.Count == 0) ? true : false;
 
                for (int j = 0; layer_included && j < layer_param.exclude.Count; j++)
                {
                    if (StateMeetsRule(net_state, layer_param.exclude[j], layer_name))
                        layer_included = false;
                }
 
                for (int j = 0; !layer_included & j < layer_param.include.Count; j++)
                {
                    if (StateMeetsRule(net_state, layer_param.include[j], layer_name))
                        layer_included = true;
                }
 
                if (layer_included)
                    param_filtered.layer.Add(layer_param.Clone(true));
            }
 
            return param_filtered;
        }
 
        public bool StateMeetsRule(NetState state, NetStateRule rule, string strLayer)
        {
            if (rule.phase == Phase.ALL)
                return true;
 
            // Check whether the rule is broken due to phase.
            if (rule.phase != Phase.NONE)
            {
                if (rule.phase != state.phase)
                {
                    m_log.WriteLine("The NetState phase (" + state.phase.ToString() + ") differed from the phase (" + rule.phase.ToString() + ") specified by a rule in layer " + strLayer);
                    return false;
                }
            }
 
            // Check whether the rule is broken due to min level.
            if (rule.min_level.HasValue)
            {
                if (state.level < rule.min_level.Value)
                {
                    m_log.WriteLine("The NetState level (" + state.level.ToString() + ") is below the min_level ( " + rule.min_level.Value.ToString() + ") specified by a rule in layer " + strLayer);
                    return false;
                }
            }
 
            // Check whether the rule is broken due to max level.
            if (rule.max_level.HasValue)
            {
                if (state.level > rule.max_level.Value)
                {
                    m_log.WriteLine("The NetState level (" + state.level.ToString() + ") is above the max_level ( " + rule.max_level.Value.ToString() + ") specified by a rule in layer " + strLayer);
                    return false;
                }
            }
 
            // Check whether the rule is broken due to stage.  The NetState must
            // contain ALL of the rule's stages to meet it.
            for (int i = 0; i < rule.stage.Count; i++)
            {
                // Check that the NetState contains the rule's ith stage.
                bool has_stage = false;
 
                for (int j = 0; !has_stage && j < state.stage.Count; j++)
                {
                    if (rule.stage[i] == state.stage[j])
                    {
                        has_stage = true;
                        break;
                    }
                }
 
                if (!has_stage)
                {
                    m_log.WriteLine("The NetState did not contain stage '" + rule.stage[i] + "' specified by a rule in layer " + strLayer);
                    return false;
                }
            }
 
            // Check whether the rule is broken due to not_stage.  The NetState must
            // contain NONE of the rule's not_stages to meet it.
            for (int i = 0; i < rule.not_stage.Count; i++)
            {
                // Check that the NetState contains the rule's ith not_stage.
                bool has_stage = false;
 
                for (int j = 0; !has_stage && j < state.stage.Count; j++)
                {
                    if (rule.not_stage[i] == state.stage[j])
                    {
                        has_stage = true;
                        break;
                    }
                }
 
                if (has_stage)
                {
                    m_log.WriteLine("The NetState contained a not_stage '" + rule.not_stage[i] + "' specified by a rule in layer " + strLayer);
                    return false;
                }
            }
 
            return true;
        }
 
        protected void AppendTop(NetParameter param, int layer_id, int top_id, List<string> available_blobs, DictionaryEx<string, int> blob_name_to_idx)
        {
            LayerParameter layer_param = null;
            string blob_name;
 
            if (layer_id >= 0)
                layer_param = param.layer[layer_id].Clone(false);
 
            if (layer_param != null)
            {
                if (layer_param.top.Count > top_id)
                    blob_name = layer_param.top[top_id];
                else
                    blob_name = "(automatic)";
            }
            else
            {
                blob_name = param.input[top_id];
            }
 
            // Check if we are doing in-place computation
            if (blob_name_to_idx != null && layer_param != null && layer_param.bottom.Count > top_id && blob_name == layer_param.bottom[top_id])
            {
                // In-place computation
                m_log.WriteLine(layer_param.name + " -> " + blob_name + " (in-place)");
                int nIdx = blob_name_to_idx[blob_name];
                m_rgcolTopVecs[layer_id].Add(m_colBlobs[nIdx]);
                m_rgrgnTopIdVecs[layer_id].Add(nIdx);
            }
            else if (blob_name_to_idx != null && blob_name_to_idx.ContainsKey(blob_name))
            {
                // If we are not doing in-place computation but have duplicated blobs,
                // raise an error.
                m_log.FAIL("Top blob '" + blob_name + "' produced by multiple sources.");
            }
            else
            {
                // Normal output.
                if (m_log.IsEnabled)
                {
                    if (layer_param != null)
                        m_log.WriteLine(layer_param.name + " -> " + blob_name);
                    else
                        m_log.WriteLine("Input " + top_id.ToString() + " -> " + blob_name);
                }
 
                Blob<T> blob_pointer = new Blob<T>(m_cuda, m_log);
                blob_pointer.Name = blob_name;
 
 
                //---------------------------------------------------
                //  When sharing this net with another (e.g. the run
                //  net shares the blobs of the training net to 
                //  help conserve gpu memory.) do not share the input 
                //  blob or the output blob for sharing the input
                //  blob will change the batch size, and sharing the
                //  output blob will cause the training layer's loss
                //  layer to be overwritten, which we do not want.
                //
                //  NOTE: The blob sharing only works when the network
                //  using the shared nodes uses sizes that are less
                //  than or equal to those of the shared node.  In
                //  the case of the run network this is not a problem
                //  for its batch size is 1 whereas the training net
                //  has a batch size of 1 or greater.
                //
                //  When sharing the training net with the testing
                //  net, blobs are only shared when the training
                //  net batch size is >= to the testing nets.
                //----------------------------------------------------
                if (m_sharedNet != null && layer_id >= 0 && layer_id < param.layer.Count() - 1)
                    m_sharedNet.blobs.Share(blob_pointer, null, false);
 
                int blob_id = m_colBlobs.Count;
                m_colBlobs.Add(blob_pointer);
                m_rgstrBlobNames.Add(blob_name);
                m_rgbBlobNeedBackward.Add(false);
 
                if (blob_name_to_idx != null)
                    blob_name_to_idx[blob_name] = blob_id;
 
                if (layer_id == -1)
                {
                    // Set the (explicitly specified) dimensions of the input blob.
                    if (param.input_dim.Count > 0)
                    {
                        blob_pointer.Reshape(param.input_dim[top_id * 4 + 0],
                                             param.input_dim[top_id * 4 + 1],
                                             param.input_dim[top_id * 4 + 2],
                                             param.input_dim[top_id * 4 + 3]);
                    }
                    else
                    {
                        blob_pointer.Reshape(param.input_shape[top_id]);
                    }
 
                    m_rgnNetInputBlobIndices.Add(blob_id);
                    m_colNetInputBlobs.Add(blob_pointer);
                }
                else
                {
                    m_rgrgnTopIdVecs[layer_id].Add(blob_id);
                    m_rgcolTopVecs[layer_id].Add(blob_pointer);
                }
            }
 
            if (available_blobs != null)
                available_blobs.Add(blob_name);
        }
 
        protected int AppendBottom(NetParameter param, int layer_id, int bottom_id, List<string> available_blobs, DictionaryEx<string, int> blob_name_to_idx)
        {
            LayerParameter layer_param = param.layer[layer_id];
            string blob_name = layer_param.bottom[bottom_id];
 
            if (!available_blobs.Contains(blob_name))
                m_log.FAIL("Unknown bottom blob '" + blob_name + "' (layer '" + layer_param.name + "', bottom index " + bottom_id.ToString() + ")");
 
            int blob_id = blob_name_to_idx[blob_name];
            m_log.WriteLine(m_rgstrLayerNames[layer_id] + " <- " + blob_name);
 
            m_rgcolBottomVecs[layer_id].Add(m_colBlobs[blob_id]);
            m_rgrgnBottomIdVecs[layer_id].Add(blob_id);
            available_blobs.Remove(blob_name);
 
            bool need_backward = m_rgbBlobNeedBackward[blob_id];
            // Check if the backpropagation on bottom_id should be skipped
            if (layer_param.propagate_down.Count > 0)
                need_backward = layer_param.propagate_down[bottom_id];
 
            m_rgrgbBottomNeedBackward[layer_id].Add(need_backward);
 
            return blob_id;
        }
 
        protected void AppendParam(NetParameter param, int layer_id, int param_id)
        {
            LayerParameter layer_param = m_rgLayers[layer_id].layer_param;
            int param_size = layer_param.parameters.Count;
            string param_name = (param_size > param_id) ? layer_param.parameters[param_id].name : "";
 
            if (param_name.Length > 0)
                m_rgstrParamDisplayNames.Add(param_name);
            else
                m_rgstrParamDisplayNames.Add(param_id.ToString());
 
            int net_param_id = m_colParams.Count;
 
            // Apply freeze learning if set.
            Blob<T> learning_param = m_rgLayers[layer_id].blobs[param_id];
            learning_param.freeze_learning = m_rgLayers[layer_id].layer_param.freeze_learning;
 
            m_colParams.Add(learning_param);
            m_rgrgnParamIdVecs[layer_id].Add(net_param_id);
            m_rgParamLayerIndices.Add(new KeyValuePair<int ,int>(layer_id, param_id));
 
            ParamSpec default_param_spec = new ParamSpec();
            ParamSpec param_spec = (layer_param.parameters.Count > param_id) ? layer_param.parameters[param_id] : default_param_spec;
 
            if (param_size == 0 || param_name.Length == 0 || (param_name.Length > 0 && !m_rgParamNamesIndex.ContainsKey(param_name)))
            {
                // This layer 'owns' this parameter blob -- it is either anonymous
                // (i.e., not given a param_name) or explicitly given a name that we
                // haven't already seen.
                m_rgnParamOwners.Add(-1);
 
                if (param_name.Length > 0)
                    m_rgParamNamesIndex[param_name] = net_param_id;
 
                int learnable_param_id = m_rgnLearnableParamIds.Count;
                m_colLearnableParams.Add(m_colParams[net_param_id]);
                m_rgnLearnableParamIds.Add(learnable_param_id);
                m_rgdfParamsLr.Add(param_spec.lr_mult);
                m_rgdfParamsWeightDecay.Add(param_spec.decay_mult);
            }
            else
            {
                // Named param blob with name we've seen before: share params
                int owner_net_param_id = m_rgParamNamesIndex[param_name];
                m_rgnParamOwners.Add(owner_net_param_id);
 
                KeyValuePair<int,int> owner_index = m_rgParamLayerIndices[owner_net_param_id];
                int owner_layer_id = owner_index.Key;
                int owner_param_id = owner_index.Value;
 
                m_log.WriteLine("Sharing parameters '" + param_name + "' owned by layer '" + m_rgstrLayerNames[owner_layer_id] + "', param index " + owner_param_id.ToString());
                
                Blob<T> this_blob = m_rgLayers[layer_id].blobs[param_id];
                Blob<T> owner_blob = m_rgLayers[owner_layer_id].blobs[owner_param_id];
                int param_size2 = layer_param.parameters.Count;
 
                if (param_size2 > param_id && (layer_param.parameters[param_id].share_mode == ParamSpec.DimCheckMode.PERMISSIVE))
                {
                    // Permissive dimension checking  -- only check counts are thet same.
                    m_log.CHECK_EQ(this_blob.count(), owner_blob.count(), "Cannot share param '" + param_name + "' owned by layer '" + m_rgstrLayerNames[owner_layer_id] + "' with layer '" + m_rgstrLayerNames[layer_id] + "'; count mismatch.  Owner layer param shape is " + owner_blob.shape_string + "; sharing layer shape is " + this_blob.shape_string);
                }
                else
                {
                    // Strict dimension checking -- all dems must be the same.
                    m_log.CHECK(Utility.Compare<int>(this_blob.shape(), owner_blob.shape()), "Cannot share param '" + param_name + "' owned by layer '" + m_rgstrLayerNames[owner_layer_id] + "' with layer '" + m_rgstrLayerNames[layer_id] + "'; shape mismatch.  Owner layer param shape is " + owner_blob.shape_string + "; sharing layer expects shape " + this_blob.shape_string);
                }
 
                int learnable_param_id = m_rgnLearnableParamIds[owner_net_param_id];
                m_rgnLearnableParamIds.Add(learnable_param_id);
 
                if (param_spec.lr_mult != 1.0)
                {
                    if (m_rgdfParamsLr[learnable_param_id].HasValue)
                        m_log.CHECK_EQ(param_spec.lr_mult, m_rgdfParamsLr[learnable_param_id].Value, "Shared param '" + param_name + "' has mismatched lr_mult.");
                    else
                        m_rgdfParamsLr[learnable_param_id] = param_spec.lr_mult;
                }
 
                if (param_spec.decay_mult != 1.0)
                {
                    if (m_rgdfParamsWeightDecay[learnable_param_id].HasValue)
                        m_log.CHECK_EQ(param_spec.decay_mult, m_rgdfParamsWeightDecay[learnable_param_id].Value, "Shared param '" + param_name + "' has mismatched decay_mult.");
                    else
                        m_rgdfParamsWeightDecay[learnable_param_id] = param_spec.decay_mult;
                }
            }
        }
 
        public double ForwardFromTo(int nStart = 0, int nEnd = int.MaxValue)
        {
            if (nEnd == int.MaxValue)
                nEnd = m_rgLayers.Count - 1;
 
            m_log.CHECK_GE(nStart, 0, "Start must be >= 0.");
            m_log.CHECK_LT(nEnd, m_rgLayers.Count, "End must be < the layer count of " + m_rgLayers.Count.ToString());
            double dfLoss = 0;
 
            for (int i = nStart; i <= nEnd; i++)
            {
                double dfLayerLoss = m_rgLayers[i].Forward(m_rgcolBottomVecs[i], m_rgcolTopVecs[i]);
                dfLoss += dfLayerLoss;
 
                if (m_bDebugInfo)
                    ForwardDebugInfo(i);
 
                //-----------------------------------------------
                // Used when debugging.
                //-----------------------------------------------
                if (m_strBestResultTargetNodeToMask != null && m_rgLayers[i].layer_param.name == m_strBestResultTargetNodeToMask)
                {
                    Blob<T> blob = blob_by_name(m_strBestResultTargetNodeToMask);
                    if (blob == null)
                        m_log.FAIL("Could not find the Best Result Target Node '" + m_strBestResultTargetNodeToMask + "'!");
 
                    if (m_nBestResultType == BEST_RESULT_TYPE.BY_CHANNEL)
                        blob.KeepBestResultsByChannel(m_nBestResultCount);
                    else
                        blob.KeepBestResultsByWeight(m_nBestResultCount);
                }
            }
 
            return dfLoss;
        }
 
        public BlobCollection<T> Forward()
        {
            double dfLoss;
            return Forward(out dfLoss);
        }
 
        public BlobCollection<T> Forward(out double dfLoss)
        {
            dfLoss = ForwardFromTo();
            return m_colNetOutputBlobs;
        }
 
        public BlobCollection<T> Forward(BlobCollection<T> colBottom, out double dfLoss, bool bReshape = false)
        {
            if (m_colNetInputBlobs.Count == 0)
            {
                if (m_bFirstForwardInputWarning)
                {
                    m_log.WriteLine("WARNING: bottom inputs are ignored, for this net does not take input.");
                    m_bFirstForwardInputWarning = false;
                }
            }
 
            // Copy bottom to internal bottom
            for (int i = 0; i < colBottom.Count && i < m_colNetInputBlobs.Count; i++)
            {
                m_colNetInputBlobs[i].CopyFrom(colBottom[i], false, bReshape);
            }
 
            return Forward(out dfLoss);
        }
 
        public void Backward(int nStart = int.MaxValue, int nEnd = 0)
        {
            if (nStart == int.MaxValue)
                nStart = m_rgLayers.Count - 1;
 
            m_log.CHECK_GE(nEnd, 0, "End must be greater than 0.");
            m_log.CHECK_LT(nStart, m_rgLayers.Count, "Start must be less than the number of layers (" + m_rgLayers.Count.ToString() + ")");
 
            for (int i = nStart; i >= nEnd; i--)
            {
                if (m_rgbLayerNeedBackward[i])
                {
                    m_rgLayers[i].Backward(m_rgcolTopVecs[i], m_rgrgbBottomNeedBackward[i], m_rgcolBottomVecs[i]);
 
                    if (m_bDebugInfo)
                        BackwardDebugInfo(i);
                }
            }
 
            if (m_bDebugInfo)
            {
                double dfAsumData = 0;
                double dfAsumDiff = 0;
                double dfSumsqData = 0;
                double dfSumsqDiff = 0;
 
                for (int i = 0; i < m_colLearnableParams.Count; i++)
                {
                    dfAsumData += Utility.ConvertVal<T>(m_colLearnableParams[i].asum_data());
                    dfAsumDiff += Utility.ConvertVal<T>(m_colLearnableParams[i].asum_diff());
                    dfSumsqData += Utility.ConvertVal<T>(m_colLearnableParams[i].sumsq_data());
                    dfSumsqDiff += Utility.ConvertVal<T>(m_colLearnableParams[i].sumsq_diff());
                }
 
                double dfL2NormData = Math.Sqrt(dfSumsqData);
                double dfL2NormDiff = Math.Sqrt(dfSumsqDiff);
 
                m_log.WriteLine("  [Backward] All net params (data, diff): L1 norm = (" + dfAsumData.ToString() + ", " + dfAsumDiff.ToString() + "; L2 norm = (" + dfL2NormData.ToString() + ", " + dfL2NormDiff.ToString() + ")");
            }
        }
 
        protected void InputDebugInfo(int input_id)
        {
            Blob<T> blob = m_colNetInputBlobs[input_id];
            int nIdx = m_rgnNetInputBlobIndices[input_id];
            string blob_name = m_rgstrBlobNames[nIdx];
            double data_abs_val_mean = Utility.ConvertVal<T>(blob.asum_data()) / blob.count();
 
            m_log.WriteLine("  [Forward] Input " + blob_name + " data: " + data_abs_val_mean.ToString());
        }
 
        protected void ForwardDebugInfo(int layer_id)
        {
            for (int top_id = 0; top_id < m_rgcolTopVecs[layer_id].Count; top_id++)
            {
                Blob<T> blob = m_rgcolTopVecs[layer_id][top_id];
                int nIdx = m_rgrgnTopIdVecs[layer_id][top_id];
                string blob_name = m_rgstrBlobNames[nIdx];
                double data_asum = Utility.ConvertVal<T>(blob.asum_data());
                double data_abs_val_mean = data_asum / blob.count();
 
                m_log.WriteLine("  [Forward] Layer " + m_rgstrLayerNames[layer_id] + ", top blob " + blob_name + " data: " + data_abs_val_mean.ToString() + " asum: " + data_asum.ToString());
            }
 
            for (int param_id = 0; param_id < m_rgLayers[layer_id].blobs.Count; param_id++)
            {
                Blob<T> blob = m_rgLayers[layer_id].blobs[param_id];
                int net_param_id = m_rgrgnParamIdVecs[layer_id][param_id];
                string blob_name = m_rgstrParamDisplayNames[net_param_id];
                double data_asum = Utility.ConvertVal<T>(blob.asum_data());
                double data_abs_val_mean = data_asum / blob.count();
 
                m_log.WriteLine("  [Forward] Layer " + m_rgstrLayerNames[layer_id] + ", param blob " + blob_name + " data: " + data_abs_val_mean.ToString() + " asum: " + data_asum.ToString());
            }
        }
 
        protected void BackwardDebugInfo(int layer_id)
        {
            BlobCollection<T> bottom_vec = m_rgcolBottomVecs[layer_id];
 
            for (int bottom_id = 0; bottom_id < bottom_vec.Count; bottom_id++)
            {
                if (!m_rgrgbBottomNeedBackward[layer_id][bottom_id])
                    continue;
 
                Blob<T> blob = bottom_vec[bottom_id];
                int nIdx = m_rgrgnBottomIdVecs[layer_id][bottom_id];
                string blob_name = m_rgstrBlobNames[nIdx];
                double diff_asum = Utility.ConvertVal<T>(blob.asum_diff());
                double diff_abs_val_mean = diff_asum / blob.count();
 
                m_log.WriteLine("  [Backward] Layer " + m_rgstrLayerNames[layer_id] + ", bottom blob " + blob_name + " diff: " + diff_abs_val_mean.ToString() + " asum: " + diff_asum.ToString());
            }
 
            for (int param_id = 0; param_id < m_rgLayers[layer_id].blobs.Count; param_id++)
            {
                if (!m_rgLayers[layer_id].param_propagate_down(param_id))
                    continue;
 
                Blob<T> blob = m_rgLayers[layer_id].blobs[param_id];
                double diff_asum = Utility.ConvertVal<T>(blob.asum_diff());
                double diff_abs_val_mean = diff_asum / blob.count();
 
                m_log.WriteLine("  [Backward] Layer " + m_rgstrLayerNames[layer_id] + ", param blob " + param_id.ToString() + " diff: " + diff_abs_val_mean.ToString() + " asum: " + diff_asum.ToString());
            }
        }
 
        protected void UpdateDebugInfo(int param_id)
        {
            Blob<T> blob = m_colBlobs[param_id];
            int param_owner = m_rgnParamOwners[param_id];
            int nIdx = m_rgParamLayerIndices[param_id].Key;
            string layer_name = m_rgstrLayerNames[nIdx];
            string param_display_name = m_rgstrParamDisplayNames[param_id];
            double diff_asum = Utility.ConvertVal<T>(blob.asum_diff());
            double diff_abs_val_mean = diff_asum / blob.count();
 
            if (param_owner < 0)
            {
                double data_abs_val_mean = Utility.ConvertVal<T>(blob.asum_data()) / blob.count();
                m_log.WriteLine("  [Update] Layer " + layer_name + ", param " + param_display_name + " data: " + data_abs_val_mean.ToString() + "; diff: " + diff_abs_val_mean.ToString() + " asum: " + diff_asum.ToString());
            }
            else
            {
                int nIdx2 = m_rgParamLayerIndices[param_owner].Key;
                string owner_layer_name = m_rgstrLayerNames[nIdx2];
                int nIdx3 = m_rgnParamOwners[param_id];
                string param_display_name_owner = m_rgstrParamDisplayNames[nIdx3];
                m_log.WriteLine("  [Update] Layer " + layer_name + ", param blob " + param_display_name + " (owned by layer " + owner_layer_name + ", param " + param_display_name_owner + ") diff: " + diff_abs_val_mean.ToString() + " asum: " + diff_asum.ToString());
            }
        }
 
        public void ShareTrainedLayersWith(Net<T> srcNet, bool bEnableLog = false)
        {
            if (srcNet == this)
                return;
 
            int num_source_layers = srcNet.layers.Count();
 
            for (int i = 0; i < num_source_layers; i++)
            {
                Layer<T> source_layer = srcNet.layers[i];
                string source_layer_name = srcNet.layer_names[i];
                int target_layer_id = 0;
 
                while (target_layer_id != m_rgstrLayerNames.Count && m_rgstrLayerNames[target_layer_id] != source_layer_name)
                {
                    target_layer_id++;
                }
 
                if (target_layer_id == m_rgstrLayerNames.Count)
                {
                    if (bEnableLog)
                        m_log.WriteLine("Ignoring source layer " + source_layer_name, true);
                    continue;
                }
 
                if (bEnableLog)
                    m_log.WriteLine("Copying source layer " + source_layer_name);
 
                BlobCollection<T> target_blobs = m_rgLayers[target_layer_id].blobs;
                m_log.CHECK_EQ(target_blobs.Count, source_layer.blobs.Count, "Incompatible number of blobs for layer " + source_layer_name);
 
                for (int j = 0; j < target_blobs.Count; j++)
                {
                    Blob<T> source_blob = source_layer.blobs[j];
                    if (!target_blobs[j].reshape_when_sharing)
                        m_log.CHECK(Utility.Compare<int>(target_blobs[j].shape(), source_blob.shape()), "Cannot share param " + j.ToString() + " weights from layer '" + source_layer_name + "'; shape mismatch.  Source param shape is " + source_blob.shape_string + "; target param shape is " + target_blobs[j].shape_string);
 
                    target_blobs[j].ShareData(source_blob);
                }
            }
        }
 
        public void CopyInternalBlobsTo(Net<T> dstNet)
        {
            m_log.CHECK_EQ(m_rgLayers.Count, dstNet.m_rgLayers.Count, "Both networks must have the same number of layers!");
 
            for (int i = 0; i < m_rgLayers.Count; i++)
            {
                m_log.CHECK_EQ(m_rgLayers[i].internal_blobs.Count, dstNet.m_rgLayers[i].internal_blobs.Count, "Both networks must have the same number of internal blobs at layer " + i.ToString());
                dstNet.m_rgLayers[i].internal_blobs.CopyFrom(m_rgLayers[i].internal_blobs);
            }
        }
 
        public void CopyTrainedLayersTo(Net<T> dstNet)
        {
            int num_source_layers = layers.Count();
 
            for (int i = 0; i < num_source_layers; i++)
            {
                Layer<T> source_layer = layers[i];
                string source_layer_name = layer_names[i];
                int target_layer_id = 0;
 
                while (target_layer_id != dstNet.m_rgstrLayerNames.Count && dstNet.m_rgstrLayerNames[target_layer_id] != source_layer_name)
                {
                    target_layer_id++;
                }
 
                if (target_layer_id == dstNet.m_rgstrLayerNames.Count)
                {
                    m_log.WriteLine("Ignoring source layer " + source_layer_name, true);
                    continue;
                }
 
                m_log.WriteLine("Copying source layer " + source_layer_name);
                BlobCollection<T> target_blobs = dstNet.m_rgLayers[target_layer_id].blobs;
                m_log.CHECK_EQ(target_blobs.Count, source_layer.blobs.Count, "Incompatible number of blobs for layer " + source_layer_name);
 
                for (int j = 0; j < target_blobs.Count; j++)
                {
                    Blob<T> source_blob = source_layer.blobs[j];
                    m_log.CHECK(Utility.Compare<int>(target_blobs[j].shape(), source_blob.shape()), "Cannot copy param " + j.ToString() + " weights from layer '" + source_layer_name + "'; shape mismatch.  Source param shape is " + source_blob.shape_string + "; target param shape is " + target_blobs[j].shape_string);
                    target_blobs[j].CopyFrom(source_blob);
                }
            }
        }
 
        public void CopyTrainedLayersTo(Net<T> dstNet, DictionaryEx<string, string> rgLayerNames, bool bTranspose)
        {
            foreach (Layer<T> sourceLayer in m_rgLayers)
            {
                string source_layer_name = sourceLayer.layer_param.name;
 
                if (rgLayerNames.ContainsKey(source_layer_name))
                {
                    string strTargetLayer = rgLayerNames[source_layer_name];
 
                    if (strTargetLayer != null && strTargetLayer.Length > 0)
                    {
                        foreach (Layer<T> targetLayer in dstNet.m_rgLayers)
                        {
                            if (targetLayer.layer_param.name == strTargetLayer)
                            {
                                m_log.WriteLine("Copying source layer " + source_layer_name);
                                BlobCollection<T> target_blobs = targetLayer.blobs;
                                m_log.CHECK_EQ(target_blobs.Count, sourceLayer.blobs.Count, "Incompatible number of blobs for layer " + source_layer_name);
                                int nCount = 1;  // currently the bias is ignored.
 
                                for (int i = 0; i < nCount; i++)
                                {
                                    Blob<T> source_blob = sourceLayer.blobs[i];
                                    m_log.CHECK(Utility.Compare<int>(target_blobs[i].shape(), source_blob.shape()), "Cannot copy param " + i.ToString() + " weights from layer '" + source_layer_name + "'; shape mismatch.  Source param shape is " + source_blob.shape_string + "; target param shape is " + target_blobs[i].shape_string);
 
                                    if (bTranspose)
                                        target_blobs[i].CopyFromAndTransposeHeightWidth(source_blob, false);
                                    else
                                        target_blobs[i].CopyFrom(source_blob, false, false);
                                }
                            }
                        }
                    }
                }
            }
        }
        
 
        public void Reshape()
        {
            for (int i = 0; i < m_rgLayers.Count; i++)
            {
                m_rgLayers[i].SetNetReshapeRequest();
                m_rgLayers[i].Reshape(m_rgcolBottomVecs[i], m_rgcolTopVecs[i]);
            }
        }
 
        public void CopyTrainedLayersFrom(NetParameter param)
        {
            int num_source_layers = param.layer.Count();
 
            for (int i = 0; i < num_source_layers; i++)
            {
                LayerParameter source_layer = param.layer[i];
                string source_layer_name = source_layer.name;
                int target_layer_id = 0;
 
                while (target_layer_id != m_rgstrLayerNames.Count && m_rgstrLayerNames[target_layer_id] != source_layer_name)
                {
                    target_layer_id++;
                }
 
                if (target_layer_id == m_rgstrLayerNames.Count)
                {
                    m_log.WriteLine("Ignoring source layer " + source_layer_name, true);
                    continue;
                }
 
                m_log.WriteLine("Copying source layer " + source_layer_name);
                BlobCollection<T> target_blobs = m_rgLayers[target_layer_id].blobs;
                m_log.CHECK_EQ(target_blobs.Count, source_layer.blobs.Count, "Incompatible number of blobs for layer " + source_layer_name);
 
                for (int j = 0; j < target_blobs.Count; j++)
                {
                    if (!target_blobs[j].ShapeEquals(source_layer.blobs[j]))
                    {
                        Blob<T> source_blob = new Blob<T>(m_cuda, m_log);
                        source_blob.FromProto(source_layer.blobs[j], true);
                        m_log.FAIL("Cannot copy param " + j.ToString() + " weights from layer " + source_layer_name + "; shape mismatch. Source param shape is " + source_blob.shape_string + "; target param shape is " + target_blobs[j].shape_string + ". To learn this layer's arameters from scratch rather than copying from the saved net, rename the layer.");
                    }
 
                    target_blobs[j].FromProto(source_layer.blobs[j], false);
                }
            }
        }
 
        public NetParameter net_param
        {
            get { return m_param; }
        }
 
        public NetParameter ToProto(bool bIncludeBlobs)
        {
            NetParameter p = m_param.Clone(true);
 
            if (bIncludeBlobs)
            {
                foreach (Layer<T> layer in m_rgLayers)
                {
                    if (layer.blobs.Count > 0)
                    {
                        foreach (LayerParameter lp in p.layer)
                        {
                            if (lp.type == layer.layer_param.type &&
                                lp.name == layer.layer_param.name)
                            {
                                foreach (Blob<T> blob in layer.blobs)
                                {
                                    lp.blobs.Add(blob.ToProto());
                                }
                            }
                        }
                    }
                }
            }
 
            return p;
        }
 
        public void Update()
        {
            for (int i = 0; i < m_colLearnableParams.Count; i++)
            {
                m_colLearnableParams[i].Update();
            }
        }
 
        public void ClearParamDiffs()
        {
            for (int i = 0; i < m_colLearnableParams.Count; i++)
            {
                Blob<T> blob = m_colLearnableParams[i];
                blob.SetDiff(0.0);
            }
        }
 
        public void ShareWeights()
        {
            for (int i = 0; i < m_colParams.Count; i++)
            {
                if (m_rgnParamOwners[i] < 0)
                    continue;
 
                int nIdx = m_rgnParamOwners[i];
                m_colParams[i].ShareData(m_colParams[nIdx]);
                m_colParams[i].ShareDiff(m_colParams[nIdx]);
            }
        }
 
        public bool ForwardBackward(BlobCollection<T> colBottom, out double dfLocalLoss, TRAIN_STEP step = TRAIN_STEP.NONE)
        {
            dfLocalLoss = 0;
 
            if (step != TRAIN_STEP.BACKWARD)
                Forward(colBottom, out dfLocalLoss);
 
            if (m_bBreakOnFirstNan)
            {
                DebugInformation<T> dbgInfo = GetDebugInformation(m_bDetectDetailedNans);
                string strType;
                string strFirstNan = dbgInfo.DetectFirstNaN(out strType);
                if (strFirstNan != null)
                    return false;
            }
 
            if (step != TRAIN_STEP.FORWARD)
            {
                Backward(int.MaxValue, m_nLastNonFrozenLayerIdx);
            }
 
            return true;
        }
 
        public string name
        {
            get { return m_strName; }
        }
 
        public List<string> layer_names
        {
            get { return m_rgstrLayerNames; }
        }
 
        public List<string> blob_names
        {
            get { return m_rgstrBlobNames; }
        }
 
        public BlobCollection<T> blobs
        {
            get { return m_colBlobs; }
        }
 
        public List<Layer<T>> layers
        {
            get { return m_rgLayers; }
        }
 
        public Phase phase
        {
            get { return m_phase; }
        }
 
        public List<BlobCollection<T>> bottom_vecs
        {
            get { return m_rgcolBottomVecs; }
        }
 
        public List<BlobCollection<T>> top_vecs
        {
            get { return m_rgcolTopVecs; }
        }
 
        public List<int> top_ids(int layer_id)
        {
            m_log.CHECK_GE(layer_id, 0, "Invalid layer id.");
            m_log.CHECK_LT(layer_id, m_rgrgnTopIdVecs.Count, "Invalid layer id.");
            return m_rgrgnTopIdVecs[layer_id];
        }
 
        public List<int> bottom_ids(int layer_id)
        {
            m_log.CHECK_GE(layer_id, 0, "Invalid layer id.");
            m_log.CHECK_LT(layer_id, m_rgrgnBottomIdVecs.Count, "Invalid layer id.");
            return m_rgrgnBottomIdVecs[layer_id];
        }
 
        public List<List<bool>> bottom_need_backward
        {
            get { return m_rgrgbBottomNeedBackward; }
        }
 
        public List<double> blob_loss_weights
        {
            get { return m_rgdfBlobLossWeights; }
        }
 
        public List<bool> layer_need_backward
        {
            get { return m_rgbLayerNeedBackward; }
        }
 
        public BlobCollection<T> parameters
        {
            get { return m_colParams; }
        }
 
        public BlobCollection<T> layer_blobs(string strLayerName)
        {
            if (!has_layer(strLayerName))
                return null;
 
            Layer<T> layer = layer_by_name(strLayerName);
 
            return layer.internal_blobs;
        }
 
        public void SetLearnedParameters(BlobCollection<T> col)
        {
            m_colLearnableParams = col;
        }
 
        public BlobCollection<T> learnable_parameters
        {
            get { return m_colLearnableParams; }
        }
 
        public List<double?> params_lr
        {
            get { return m_rgdfParamsLr; }
        }
 
        public List<double?> params_weight_decay
        {
            get { return m_rgdfParamsWeightDecay; }
        }
 
        public DictionaryEx<string, int> param_names_index
        {
            get { return m_rgParamNamesIndex; }
        }
 
        public List<int> param_owners
        {
            get { return m_rgnParamOwners; }
        }
 
        public List<string> param_display_names
        {
            get { return m_rgstrParamDisplayNames; }
        }
 
        public Blob<T> param_by_name(string strName, bool bThrowExceptionOnError = true)
        {
            foreach (Blob<T> blob in m_colParams)
            {
                if (blob.Name == strName)
                    return blob;
            }
 
            if (bThrowExceptionOnError)
                m_log.FAIL("Unknown parameter blob name '" + strName + "'");
 
            return null;
        }
 
        public int num_inputs
        {
            get { return m_colNetInputBlobs.Count; }
        }
 
        public int num_outputs
        {
            get { return m_colNetOutputBlobs.Count; }
        }
 
        public BlobCollection<T> input_blobs
        {
            get { return m_colNetInputBlobs; }
        }
 
        public BlobCollection<T> output_blobs
        {
            get { return m_colNetOutputBlobs; }
        }
 
        public List<int> output_blob_indices
        {
            get { return m_rgnNetOutputBlobIndices; }
        }
 
        public List<int> input_blob_indices
        {
            get { return m_rgnNetInputBlobIndices; }
        }
 
        public bool has_blob(string strBlobName)
        {
            return m_rgBlobNamesIndex.ContainsKey(strBlobName);
        }
 
        public Blob<T> blob_by_name(string strName, bool bThrowExceptionOnError = true)
        {
            Blob<T> blob_ptr = null;
 
            if (has_blob(strName))
            {
                int nIdx = m_rgBlobNamesIndex[strName];
                blob_ptr = m_colBlobs[nIdx];
            }
            else
            {
                if (bThrowExceptionOnError)
                    m_log.FAIL("Unknown blob name " + strName);
            }
 
            return blob_ptr;
        }
 
        public int blob_index_by_name(string strName)
        {
            if (!has_blob(strName))
                return -1;
 
            return m_rgBlobNamesIndex[strName];
        }
 
        public bool has_layer(string strLayer)
        {
            return m_rgLayerNamesIndex.ContainsKey(strLayer);
        }
 
        public Layer<T> layer_by_name(string strLayer, bool bThrowExceptionOnError = true)
        {
            Layer<T> layer_ptr = null;
 
            if (has_layer(strLayer))
            {
                int nIdx = m_rgLayerNamesIndex[strLayer];
                layer_ptr = m_rgLayers[nIdx];
            }
            else
            {
                if (bThrowExceptionOnError)
                    m_log.FAIL("Unknown layer name " + strLayer);
            }
 
            return layer_ptr;
        }
 
        public int layer_index_by_name(string strLayer)
        {
            if (!has_layer(strLayer))
                return -1;
 
            return m_rgLayerNamesIndex[strLayer];
        }
 
        public void set_debug_info(bool bVal)
        {
            m_bDebugInfo = bVal;
        }
 
 
 
        public NetParameter InsertSplits(NetParameter param)
        {
            // Initialize by copying from the input NetParameter.
            NetParameter param_split = param.Clone(false);
 
            DictionaryEx<string, KeyValuePair<int, int>> blob_name_to_last_top_idx = new DictionaryEx<string,KeyValuePair<int,int>>(new KeyValuePair<int,int>(-1, -1));
            DictionaryEx<KeyValuePair<int, int>, KeyValuePair<int, int>> bottom_idx_to_source_top_idx = new DictionaryEx<KeyValuePair<int,int>,KeyValuePair<int,int>>(new KeyValuePair<int,int>(-1, -1));
            DictionaryEx<KeyValuePair<int, int>, int> top_idx_to_bottom_count = new DictionaryEx<KeyValuePair<int,int>,int>(0);
            DictionaryEx<KeyValuePair<int, int>, double> top_idx_to_loss_weight = new DictionaryEx<KeyValuePair<int,int>,double>(0);
            DictionaryEx<KeyValuePair<int, int>, int> top_idx_to_bottom_split_idx = new DictionaryEx<KeyValuePair<int,int>,int>(0);
            DictionaryEx<int, string> layer_idx_to_layer_name = new DictionaryEx<int,string>("");
 
            layer_idx_to_layer_name[-1] = "input";
 
            // Determine the number of times each blob is used as an input (bottom) blob.
 
            for (int i = 0; i < param.input.Count; i++)
            {
                string blob_name = param.input[i];
                blob_name_to_last_top_idx[blob_name] = new KeyValuePair<int, int>(-1, i);
            }
 
            for (int i = 0; i < param.layer.Count; i++)
            {
                LayerParameter layer_param = param.layer[i];
                layer_idx_to_layer_name[i] = layer_param.name;
 
                for (int j = 0; j < layer_param.bottom.Count; j++)
                {
                    string blob_name = layer_param.bottom[j];
 
                    if (!blob_name_to_last_top_idx.ContainsKey(blob_name))
                        m_log.FAIL("Unknown bottom blob '" + blob_name + "' (layer '" + layer_param.name + "', bottom index " + j.ToString() + ")");
 
                    KeyValuePair<int, int> bottom_idx = new KeyValuePair<int, int>(i, j);
                    KeyValuePair<int, int> top_idx = blob_name_to_last_top_idx[blob_name];
                    bottom_idx_to_source_top_idx[bottom_idx] = top_idx;
                    top_idx_to_bottom_count[top_idx]++;
                }
 
                for (int j = 0; j < layer_param.top.Count; j++)
                {
                    string blob_name = layer_param.top[j];
                    blob_name_to_last_top_idx[blob_name] = new KeyValuePair<int, int>(i, j);
                }
 
                // A use of a top blob as a loss should be handled similarly to the use of
                // a top blob as an input (bottom) blob to another layer.
                int last_loss = Math.Min(layer_param.loss_weight.Count, layer_param.top.Count);
 
                for (int j = 0; j < last_loss; j++)
                {
                    string blob_name = layer_param.top[j];
                    KeyValuePair<int, int> top_idx = blob_name_to_last_top_idx[blob_name];
                    top_idx_to_loss_weight[top_idx] = layer_param.loss_weight[j];
 
                    if (top_idx_to_loss_weight[top_idx] != 0)
                        top_idx_to_bottom_count[top_idx]++;
                }
            }
 
            // Create split layer for any input blobs used by other layer as bottom
            // blobs more than once.
            for (int i = 0; i < param.input.Count; i++)
            {
                int split_count = top_idx_to_bottom_count[new KeyValuePair<int, int>(-1, i)];
 
                if (split_count > 1)
                {
                    string layer_name = layer_idx_to_layer_name[-1];
                    string blob_name = param.input[i];
                    double kZeroLossWeight = 0;
                    LayerParameter split_layer_param = CreateSplitLayer(layer_name, blob_name, i, split_count, kZeroLossWeight);
                    param_split.layer.Add(split_layer_param);
                }
            }
 
            for (int i = 0; i < param.layer.Count; i++)
            {
                LayerParameter layer_param = param.layer[i].Clone(true);
                param_split.layer.Add(layer_param);
 
                // Replace any shared bottom blobs with split layer outputs.
                for (int j = 0; j < layer_param.bottom.Count; j++)
                {
                    KeyValuePair<int, int> top_idx = bottom_idx_to_source_top_idx[new KeyValuePair<int, int>(i, j)];
                    int split_count = top_idx_to_bottom_count[top_idx];
 
                    if (split_count > 1)
                    {
                        string layer_name = layer_idx_to_layer_name[top_idx.Key];
                        string blob_name = layer_param.bottom[j];
 
                        layer_param.bottom[j] = SplitBlobName(layer_name, blob_name, top_idx.Value, top_idx_to_bottom_split_idx[top_idx]++);
                    }
                }
 
                // Create split layer for any top blobs used by other layer as bottom
                // blobs more than once.
                for (int j = 0; j < layer_param.top.Count; j++)
                {
                    KeyValuePair<int, int> top_idx = new KeyValuePair<int, int>(i, j);
                    int split_count = top_idx_to_bottom_count[top_idx];
 
                    if (split_count > 1)
                    {
                        string layer_name = layer_idx_to_layer_name[i];
                        string blob_name = layer_param.top[j];
                        double loss_weight = top_idx_to_loss_weight[top_idx];
                        LayerParameter split_layer_param = CreateSplitLayer(layer_name, blob_name, j, split_count, loss_weight);
                        param_split.layer.Add(split_layer_param);
 
                        if (loss_weight != 0)
                        {
                            layer_param.loss_weight.Clear();
                            top_idx_to_bottom_split_idx[top_idx]++;
                        }
                    }
                }
            }
 
            return param_split;
        }
 
        private LayerParameter CreateSplitLayer(string layer_name, string blob_name, int blob_idx, int split_count, double loss_weight)
        {
            LayerParameter split_layer_param = new LayerParameter(LayerParameter.LayerType.SPLIT, SplitLayerName(layer_name, blob_name, blob_idx));
            split_layer_param.bottom.Add(blob_name);
 
            for (int k = 0; k < split_count; k++)
            {
                split_layer_param.top.Add(SplitBlobName(layer_name, blob_name, blob_idx, k));
 
                if (loss_weight != 0)
                {
                    if (k == 0)
                        split_layer_param.loss_weight.Add(loss_weight);
                    else
                        split_layer_param.loss_weight.Add(0);
                }
            }
 
            return split_layer_param;
        }
 
        private string SplitLayerName(string layer_name, string blob_name, int blob_idx)
        {
            return blob_name + "_" + layer_name + "_" + blob_idx.ToString() + "_split";
        }
 
        private string SplitBlobName(string layer_name, string blob_name, int blob_idx, int split_idx)
        {
            return blob_name + "_" + layer_name + "_" + blob_idx.ToString() + "_split_" + split_idx.ToString();
        }
 
        public void LoadWeights(byte[] rgWeights, IXPersist<T> persist, List<string> inputWtInfo = null, List<string> targetWtInfo = null, string strSkipBlobType = null)
        {
            if (rgWeights == null)
                return;
 
            List<string> rgExpectedShapes = new List<string>();
            bool bLoadedDiffs;
 
            foreach (Blob<T> b in m_colLearnableParams)
            {
                rgExpectedShapes.Add(b.shape_string);
            }
 
            if (inputWtInfo != null && inputWtInfo.Count == 0)
                inputWtInfo = null;
 
            if (targetWtInfo != null && targetWtInfo.Count == 0)
                targetWtInfo = null;
 
            bool bSizeToFit = (inputWtInfo != null && targetWtInfo != null) ? true : false;
 
            persist.LoadWeights(rgWeights, rgExpectedShapes, m_colLearnableParams, bSizeToFit, out bLoadedDiffs, inputWtInfo, targetWtInfo, strSkipBlobType);
            m_cuda.SynchronizeDevice();
        }
 
        public byte[] SaveWeights(IXPersist<T> persist, bool bSaveDiff = false)
        {
            foreach (Blob<T> blob in m_colLearnableParams)
            {
                foreach (Layer<T> layer in m_rgLayers)
                {
                    if (layer.blobs.Contains(blob))
                        blob.Tag = layer.layer_param.name;
                }
            }
 
            return persist.SaveWeights(m_colLearnableParams, bSaveDiff);
        }
 
        public Layer<T> FindLayerOwningBlob(Blob<T> b)
        {
            foreach (Layer<T> layer in layers)
            {
                if (layer.blobs.Contains(b))
                    return layer;
            }
 
            return null;
        }
 
        public int FindLayerIndexOwningBlob(Blob<T> b)
        {
            for (int i = 0; i < layers.Count; i++)
            {
                if (layers[i].blobs.Contains(b))
                    return i;
            }
 
            return -1;
        }
 
        public Blob<T> FindBlob(string strName)
        {
            foreach (Blob<T> blob in blobs)
            {
                if (blob.Name == strName)
                    return blob;
            }
 
            foreach (Layer<T> layer in m_rgLayers)
            {
                foreach (Blob<T> blob in layer.blobs)
                {
                    if (blob.Name == strName)
                        return blob;
                }
 
                foreach (Blob<T> blob in layer.internal_blobs)
                {
                    if (blob.Name == strName)
                        return blob;
                }
            }
 
            return null;
        }
 
        public BlobCollection<T> FindBottomBlobsOfLayer(string strLayer)
        {
            int nLayerIdx = 0;
 
            for (int i = 0; i < m_rgLayers.Count; i++)
            {
                if (m_rgLayers[i].layer_param.name == strLayer)
                {
                    nLayerIdx = i;
                    break;
                }
            }
 
            return m_rgcolBottomVecs[nLayerIdx];
        }
 
        public BlobCollection<T> FindTopBlobsOfLayer(string strLayer)
        {
            int nLayerIdx = 0;
 
            for (int i = 0; i < m_rgLayers.Count; i++)
            {
                if (m_rgLayers[i].layer_param.name == strLayer)
                {
                    nLayerIdx = i;
                    break;
                }
            }
 
            return m_rgcolTopVecs[nLayerIdx];
        }
 
        public Blob<T> FindLossBottomBlob()
        {
            if (m_rgLayers.Count == 0)
                return null;
 
            int nLayerIdx = m_rgLayers.Count - 1;
            Layer<T> layer = m_rgLayers[nLayerIdx];
 
            while (nLayerIdx > 0 && (!layer.parent_layer_type.HasValue || layer.parent_layer_type.Value != LayerParameter.LayerType.LOSS))
            {
                nLayerIdx--;
                layer = m_rgLayers[nLayerIdx];
            }
 
            if (nLayerIdx > 0)
            {
                string strBtm = layer.layer_param.bottom[0];
                return blob_by_name(strBtm);
            }
 
            return null;
        }
 
        public DebugInformation<T> GetDebugInformation(bool bDetectNans)
        {
            if (m_blobWork == null)
                m_blobWork = new common.Blob<T>(m_cuda, m_log);
 
            DebugInformation<T> debugInfo = new DebugInformation<T>(name, m_blobWork, bDetectNans);
 
            for (int i = 0; i < m_rgLayers.Count; i++)
            {
                debugInfo.Add(m_rgLayers[i], m_rgcolBottomVecs[i], m_rgcolTopVecs[i]);
            }
 
            return debugInfo;
        }
 
        public string GetDataSource()
        {
            foreach (LayerParameter lp in m_param.layer)
            {
                if (lp.type == LayerParameter.LayerType.DATA)
                    return lp.data_param.source;
            }
 
            return null;
        }
 
        public bool ReInitializeParameters(WEIGHT_TARGET target, params string[] rgstrLayers)
        {
            foreach (Layer<T> layer in m_rgLayers)
            {
                if (rgstrLayers == null || rgstrLayers.Length == 0 || rgstrLayers.Contains(layer.layer_param.name))
                {
                    layer.ReInitializeParameters(target);
                }
            }
 
            return true;
        }
 
        public Layer<T> FindLayer(LayerParameter.LayerType? type, string strName)
        {
            if (!type.HasValue && string.IsNullOrEmpty(strName))
                throw new Exception("You must specify either a layer type or name, or both.");
 
            foreach (Layer<T> layer in m_rgLayers)
            {
                bool bTypeMatch = !type.HasValue;
                bool bNameMatch = string.IsNullOrEmpty(strName);
 
                if (type.HasValue && layer.type == type.Value)
                    bTypeMatch = true;
 
                if (!string.IsNullOrEmpty(strName) && layer.layer_param.name == strName)
                    bNameMatch = true;
 
                if (bTypeMatch && bNameMatch)
                    return layer;
            }
 
            return null;
        }
 
        public List<Layer<T>> FindLayers(LayerParameter.LayerType type)
        {
            List<Layer<T>> rgLayers = new List<Layer<T>>();
 
            foreach (Layer<T> layer in m_rgLayers)
            {
                if (layer.type == type)
                    rgLayers.Add(layer);
            }
 
            return rgLayers;
        }
 
        public Layer<T> FindLayer(string strType, string strName)
        {
            LayerParameter.LayerType? type = LayerParameter.GetType(strType);
            return FindLayer(type, strName);
        }
 
        public Layer<T> FindLastLayer(LayerParameter.LayerType type)
        {
            for (int i = m_rgLayers.Count - 1; i >= 0; i--)
            {
                if (m_rgLayers[i].type == type)
                    return m_rgLayers[i];
            }
 
            return null;
        }
 
        public CudaDnn<T> Cuda
        {
            get { return m_cuda; }
        }
    }
}