AdaGradSolver.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.IO;
using MyCaffe.basecode;
using MyCaffe.db.image;
using MyCaffe.common;
using MyCaffe.param;
 
namespace MyCaffe.solvers
{
    public class AdaGradSolver<T> : SGDSolver<T>
    {
        public AdaGradSolver(CudaDnn<T> cuda, Log log, SolverParameter p, CancelEvent evtCancel, AutoResetEvent evtForceSnapshot, AutoResetEvent evtForceTest, IXDatabaseBase db, IXPersist<T> persist, int nSolverCount = 1, int nSolverRank = 0, Net<T> shareNet = null, onGetWorkspace getws = null, onSetWorkspace setws = null)
            : base(cuda, log, p, evtCancel, evtForceSnapshot, evtForceTest, db, persist, nSolverCount, nSolverRank, shareNet, getws, setws)
        {
            m_log.CHECK_EQ(0, m_param.momentum, "Momentum cannot be used with AdaGrad.");
        }
 
        public override void ComputeUpdateValue(int param_id, double dfRate, int nIterationOverride = -1)
        {
            BlobCollection<T> colNetParams = m_net.learnable_parameters;
 
            if (!colNetParams[param_id].DiffExists)
                return;
 
            List<double?> net_params_lr = m_net.params_lr;
            T fDelta = Utility.ConvertVal<T>(m_param.delta);
            T fLocalRate = Utility.ConvertVal<T>(dfRate * net_params_lr[param_id].GetValueOrDefault(0));
 
            // Compute the update to history, then copy it to the parameter diff.
            m_cuda.adagrad_update(colNetParams[param_id].count(), 
                                  colNetParams[param_id].mutable_gpu_diff, 
                                  m_colHistory[param_id].mutable_gpu_data, 
                                  fDelta, 
                                  fLocalRate);
        }
    }
}