TokenizedDataPairsLayer.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using MyCaffe.basecode;
using MyCaffe.common;
using MyCaffe.param;
using MyCaffe.fillers;
using System.IO;
using MyCaffe.db.image;
using MyCaffe.param.gpt;
using System.Net;
using System.Globalization;
using System.Diagnostics;
using System.Xml.Linq;
using System.Security.Cryptography;
using System.Threading;
using System.IO.Compression;
using System.Reflection;
 
namespace MyCaffe.layers.gpt
{
    public class TokenizedDataPairsLayer<T> : Layer<T>
    {
        CancelEvent m_evtCancel;
        InputData m_encoderData = null;
        InputData m_decoderData = null;
        Blob<T> m_blobX = null;
        Blob<T> m_blobY = null;
        Blob<T> m_blobTriangle = null;
        Random m_random = new Random();
        Blob<T> m_blobEncIn = null;
        Blob<T> m_blobDecIn = null;
        Layer<T> m_softmax = null;
        Layer<T> m_argmax = null;
        Stopwatch m_swUpdateTimer = new Stopwatch();
        double m_dfLastProgress = 0;
        AutoResetEvent m_evtDownloadDone = new AutoResetEvent(false);
 
        public enum VOCABULARY
        {
            ENCODER,
            DECODER
        }
 
        public TokenizedDataPairsLayer(CudaDnn<T> cuda, Log log, LayerParameter p, IXDatabaseBase db, CancelEvent evtCancel)
            : base(cuda, log, p)
        {
            m_evtCancel = evtCancel;
            m_type = LayerParameter.LayerType.TOKENIZED_DATA_PAIRS;
 
            m_blobTriangle = new Blob<T>(m_cuda, m_log, false);
            m_blobTriangle.Name = "triangle";
        }
 
        protected override void dispose()
        {
            dispose(ref m_blobY);
            dispose(ref m_blobX);
            dispose(ref m_blobTriangle);
            dispose(ref m_blobEncIn);
            dispose(ref m_blobDecIn);
            dispose(ref m_softmax);
            dispose(ref m_argmax);
 
            base.dispose();
        }
 
        protected override void setup_internal_blobs(BlobCollection<T> col)
        {
            col.Add(m_blobTriangle);
        }
 
        public override int ExactNumBottomBlobs
        {
            get { return (m_phase == Phase.RUN) ? 2 : 0; }
        }
 
        public override int ExactNumTopBlobs
        {
            get { return 5; }
        }
 
        public override void LayerSetUp(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            switch (m_param.tokenized_data_pairs_param.input_type)
            {
                case TokenizedDataParameter.INPUT_TYPE.TEXT_FILE:
                    download_vocab_data();
                    m_encoderData = new TextListData(m_log, m_param.tokenized_data_pairs_param.source, m_param.tokenized_data_pairs_param.source_vocab_file, false, m_param.tokenized_data_pairs_param.vocabulary_type, m_param.tokenized_data_pairs_param.seed, m_param.phase);
                    m_decoderData = new TextListData(m_log, m_param.tokenized_data_pairs_param.target, m_param.tokenized_data_pairs_param.target_vocab_file, true, m_param.tokenized_data_pairs_param.vocabulary_type, m_param.tokenized_data_pairs_param.seed, m_param.phase);
                    m_log.WriteLine("Encoder Vocabulary: " + m_encoderData.VocabularySize.ToString());
                    m_log.WriteLine("Decoder Vocabulary: " + m_decoderData.VocabularySize.ToString());
                    break;
 
                case TokenizedDataParameter.INPUT_TYPE.CUSTOM:
                    m_encoderData = new CustomListData(m_evtCancel, m_log, m_param.tokenized_data_pairs_param.source, "ENC", (int)m_param.tokenized_data_pairs_param.block_size, m_param.tokenized_data_pairs_param.seed, m_param.phase);
                    m_decoderData = new CustomListData(m_evtCancel, m_log, m_param.tokenized_data_pairs_param.source, "DEC", (int)m_param.tokenized_data_pairs_param.block_size, m_param.tokenized_data_pairs_param.seed, m_param.phase);
                    break;
 
                default:
                    throw new Exception("Unknown input type '" + m_param.tokenized_data_pairs_param.input_type.ToString() + "'");
            }
        }
 
        private void download_vocab_data()
        {
            if (string.IsNullOrEmpty(m_param.tokenized_data_pairs_param.vocab_data_url))
                return;
 
            string strProgData = Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData);
            string strDataFile = Utility.ReplaceMacro(m_param.tokenized_data_pairs_param.vocab_data_dst_file, "$ProgramData$", strProgData);
            string strVocabFile = Utility.ReplaceMacro(m_param.tokenized_data_pairs_param.source, "$ProgramData$", strProgData);
 
            if (string.IsNullOrEmpty(strDataFile))
                m_log.FAIL("You must specify a 'vocab_data_dst_file' when using 'vocab_data_url'.");
 
            string strPath = Path.GetDirectoryName(strDataFile);
            if (!Directory.Exists(strPath))
                Directory.CreateDirectory(strPath);
 
            string strFile1 = Path.GetFileName(strDataFile);
            string strFile = downloadData(m_param.tokenized_data_pairs_param.vocab_data_url, strPath, strFile1);
            if (!File.Exists(strFile))
                return;
 
            if (!File.Exists(strVocabFile))
                ZipFile.ExtractToDirectory(strFile, strPath);
        }
 
        private string downloadData(string strUrl, string strPath, string strFileName)
        {
            if (!Directory.Exists(strPath))
                Directory.CreateDirectory(strPath);
 
            string strDataFile = strPath + "\\" + strFileName;
            if (!File.Exists(strDataFile))
            {
                using (WebClient webClient = new WebClient())
                {
                    string strFile1 = strFileName;
                    string strFile = strPath + "\\" + strFile1;
 
                    m_swUpdateTimer.Start();
                    m_dfLastProgress = 0;
 
                    webClient.DownloadProgressChanged += WebClient_DownloadProgressChanged;
                    webClient.DownloadFileCompleted += WebClient_DownloadFileCompleted;
                    webClient.DownloadFileAsync(new Uri(strUrl), strFile, strFile1);
 
                    m_evtDownloadDone.WaitOne();
                }
            }
 
            return strDataFile;
        }
 
        private void WebClient_DownloadFileCompleted(object sender, System.ComponentModel.AsyncCompletedEventArgs e)
        {
            m_log.WriteLine("Downloading done.", true);
            m_evtDownloadDone.Set();
        }
 
        private void WebClient_DownloadProgressChanged(object sender, DownloadProgressChangedEventArgs e)
        {
            if (m_swUpdateTimer.Elapsed.TotalMilliseconds >= 1000)
            {
                if (m_dfLastProgress != e.ProgressPercentage)
                {
                    m_dfLastProgress = e.ProgressPercentage;
                    string strFile = e.UserState.ToString();
 
                    double dfPct = e.ProgressPercentage / 100.0;
                    m_log.WriteLine("Downloading '" + strFile + "' at " + dfPct.ToString("P") + "...", true);
                }
 
                m_swUpdateTimer.Restart();
            }
        }
 
        public override void Reshape(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            int nBatchSize = (int)m_param.tokenized_data_pairs_param.batch_size;
            int nBlockSize = (int)m_param.tokenized_data_pairs_param.block_size;
            int nTokenSize = (int)m_encoderData.TokenSize;
 
            if (m_phase == Phase.RUN)
                nBatchSize = 1;
            else
                m_log.CHECK_EQ(colBottom.Count, 0, "Data Layer takes no input blobs.");
             
            m_log.CHECK_EQ(colTop.Count, 5, "The TokenizedDataPairsLayer requires 5 top blobs.");
 
            Blob<T> blobEncIn = colTop[0];
            Blob<T> blobDecIn = colTop[1];
            Blob<T> blobDecOut = colTop[2];
            Blob<T> blobEncMask = colTop[3];
            Blob<T> blobDecMask = colTop[4];
 
            int nCount = 3;
            if (nTokenSize == 1)
                nCount = 2;
            int[] rgShape = new int[nCount];
 
            blobEncIn.SetParameter("vocab_size", m_encoderData.VocabularySize);
            blobDecIn.SetParameter("vocab_size", m_decoderData.VocabularySize);
            // reshape for single characters (each character is an index into the vocab vector)
            rgShape[0] = nBatchSize;
            rgShape[1] = nBlockSize;
            if (rgShape.Length > 2)
                rgShape[2] = nTokenSize;
 
            blobEncIn.Reshape(rgShape);
            blobDecIn.Reshape(rgShape);
            blobDecOut.Reshape(rgShape);
            blobEncMask.Reshape(nBatchSize, nBlockSize, 1, 1);
            blobDecMask.Reshape(nBatchSize, nBlockSize, nBlockSize, 1);
 
            if (!m_blobTriangle.CompareShape(blobDecMask.shape()))
            {
                m_blobTriangle.ReshapeLike(blobDecMask);
 
                T[] rgMask = new T[m_blobTriangle.count()];
                for (int n = 0; n < m_blobTriangle.num; n++)
                {
                    for (int c = 0; c < m_blobTriangle.channels; c++)
                    {
                        for (int h = 0; h < m_blobTriangle.height; h++)
                        {
                            int nIdx = n * nBlockSize * nBlockSize + c * nBlockSize + h;
                            rgMask[nIdx] = (h > c) ? m_tZero : m_tOne;
                        }
                    }
                }
 
                m_blobTriangle.mutable_cpu_data = rgMask;
            }
        }
 
        protected override void forward(BlobCollection<T> colBottom, BlobCollection<T> colTop)
        {
            if (m_phase == Phase.RUN)
            {
                colTop[0].CopyFromAndPad(colBottom[0]); // enc data
                colTop[1].CopyFromAndPad(colBottom[1]); // dec data (with BOS token, initially)
                // colTop[2] NO Dec target data when running.
            }
            else
            {
                int[] rgnIdx;
                Tuple<float[], float[]> encData = m_encoderData.GetData((int)m_param.tokenized_data_pairs_param.batch_size, (int)m_param.tokenized_data_pairs_param.block_size, m_decoderData, out rgnIdx);
                Tuple<float[], float[]> decData = m_decoderData.GetDataAt((int)m_param.tokenized_data_pairs_param.batch_size, (int)m_param.tokenized_data_pairs_param.block_size, rgnIdx);
 
                colTop[0].mutable_cpu_data = convert(encData.Item1);
                colTop[1].mutable_cpu_data = convert(decData.Item1);
                colTop[2].mutable_cpu_data = convert(decData.Item2);
            }
 
            // Fill encoder mask based on encoder input.
            m_cuda.sign(colTop[0].count(), colTop[0].gpu_data, colTop[3].mutable_gpu_data);
            // Fill decoder mask based on decoder input.
            m_cuda.channel_duplicate(colTop[4].count(), colTop[1].num, colTop[1].channels, colTop[4].count(2), colTop[1].gpu_data, colTop[4].mutable_gpu_data);
            m_cuda.sign(colTop[4].count(), colTop[4].gpu_data, colTop[4].mutable_gpu_data);
            // Overlay triangular matrix on decoder mask.
            m_cuda.mul(colTop[4].count(), colTop[4].gpu_data, m_blobTriangle.gpu_data, colTop[4].mutable_gpu_data);
        }
 
        protected override void backward(BlobCollection<T> colTop, List<bool> rgbPropagateDown, BlobCollection<T> colBottom)
        {
        }
 
        public override bool SupportsPreProcessing
        {
            get { return true; }
        }
 
        public override bool SupportsPostProcessingLogits
        {
            get { return true; }
        }
 
        public List<int> Tokenize(string str, VOCABULARY vocab)
        {
            if (vocab == VOCABULARY.ENCODER)
                return m_encoderData.Tokenize(str, false, false);
            else
                return m_decoderData.Tokenize(str, false, false);
        }
 
        public string Detokenize(float[] rg, int nStartIdx, int nCount, VOCABULARY vocab)
        {
            InputData inputData = (vocab == VOCABULARY.ENCODER) ? m_encoderData : m_decoderData;
            return inputData.Detokenize(rg, nStartIdx, nCount, false, false);
        }
 
        public uint GetVocabuarySize(VOCABULARY src)
        {
            InputData input = (src == VOCABULARY.ENCODER) ? m_encoderData : m_decoderData;
            return input.VocabularySize;
        }
 
        public override BlobCollection<T> PreProcessInput(PropertySet customInput, out int nSeqLen, BlobCollection<T> colBottom = null)
        {
            nSeqLen = (int)m_param.tokenized_data_pairs_param.block_size;
 
            if (m_blobEncIn == null)
                m_blobEncIn = new Blob<T>(m_cuda, m_log);
 
            if (m_blobDecIn == null)
                m_blobDecIn = new Blob<T>(m_cuda, m_log);
 
            string strInput = customInput.GetProperty("InputData");
            if (string.IsNullOrEmpty(strInput))
                throw new Exception("Could not find 'InputData' property!");
 
            List<int> rgTokens = m_encoderData.Tokenize(strInput, true, true);
            float[] rgInput = new float[rgTokens.Count];
 
            for (int i = 0; i < rgTokens.Count; i++)
            {
                rgInput[i] = rgTokens[i];
            }
 
            int[] rgShape = new int[2];
            rgShape[0] = 1;
            rgShape[1] = rgInput.Length;
 
            m_blobEncIn.Reshape(rgShape);
 
            rgShape[1] = 1;
            m_blobDecIn.Reshape(rgShape);
 
            m_blobEncIn.mutable_cpu_data = convert(rgInput);
            m_blobDecIn.SetData((int)SPECIAL_TOKENS.BOS);
 
            return new BlobCollection<T>() { m_blobEncIn, m_blobDecIn };
        }
 
        public override bool PreProcessInput(string str, int? nTokIdx, BlobCollection<T> colBottom = null)
        {
            if (nTokIdx.HasValue && nTokIdx.Value == (int)SPECIAL_TOKENS.EOS)
                return false;
 
            Blob<T> blobBtm = (colBottom.Count > 1) ? colBottom[1] : colBottom[0];
 
            List<float> rgTok = convertF(blobBtm.mutable_cpu_data).ToList();
 
            rgTok.Add(nTokIdx.Value);
            if (rgTok.Count > m_param.tokenized_data_pairs_param.block_size)
                rgTok.RemoveAt(0);
 
            List<int> rgShape = Utility.Clone<int>(blobBtm.shape());
            rgShape[1] = rgTok.Count;
            blobBtm.Reshape(rgShape);
 
            blobBtm.mutable_cpu_data = convert(rgTok.ToArray());
 
            return true;
        }
 
        public override List<Tuple<string, int, double>> PostProcessLogitsOutput(int nCurIdx, Blob<T> blobLogits, Layer<T> softmax, int nAxis, int nK = 1)
        {
            float[] rgData = convertF(blobLogits.mutable_cpu_data);
            int nVocabCount = blobLogits.count(nAxis);
            float[] rgLogits = new float[nVocabCount];
            Dictionary<int, float> rgTopK = new Dictionary<int, float>();
            
            if (m_blobX == null)
                m_blobX = new Blob<T>(m_cuda, m_log);
            if (m_blobY == null)
                m_blobY = new Blob<T>(m_cuda, m_log);
 
            BlobCollection<T> colBottom = new BlobCollection<T>() { blobLogits };
            BlobCollection<T> colTop = new BlobCollection<T>() { m_blobY };
            if (softmax == null)
            {
                if (m_softmax == null)
                {
                    LayerParameter softmax_param = new LayerParameter(LayerParameter.LayerType.SOFTMAX);
                    softmax_param.softmax_param.axis = nAxis;
                    m_softmax = Layer<T>.Create(m_cuda, m_log, softmax_param, null);
                    m_softmax.Setup(colBottom, colTop);
                }
 
                softmax = m_softmax;
            }
            
            if (m_argmax == null)
            {
                LayerParameter argmax_param = new LayerParameter(LayerParameter.LayerType.ARGMAX);
                argmax_param.argmax_param.out_max_val = false;
                argmax_param.argmax_param.enable_cuda_impl = true;
                argmax_param.argmax_param.axis = nAxis;
                m_argmax = Layer<T>.Create(m_cuda, m_log, argmax_param, null);
                softmax.Reshape(colBottom, colTop);
                m_blobX.ReshapeLike(colTop[0]);
                colBottom[0] = m_blobX;
                m_argmax.Setup(colBottom, colTop);
            }
 
            colBottom[0] = blobLogits;
            softmax.Forward(colBottom, colTop);
            m_blobX.CopyFrom(colTop[0]);
            colBottom[0] = m_blobX;
            m_argmax.Forward(colBottom, colTop);
 
            float[] rgArgMax = convertF(colTop[0].mutable_cpu_data);
            int nCharIdx = (int)rgArgMax[nCurIdx];
 
            string str = m_decoderData.Detokenize(nCharIdx, true, true);
            str += " ";
 
            return new List<Tuple<string, int, double>>() { new Tuple<string, int, double>(str, nCharIdx, 0) };
        }
 
        public override string PostProcessFullOutput(Blob<T> blobSoftmax)
        {
            float[] rgData = convertF(blobSoftmax.mutable_cpu_data);
            string strOut = "";
 
            foreach (float fTok in rgData)
            {
                if (fTok == 0)
                    break;
 
                strOut += m_decoderData.Detokenize((int)fTok, true, true);
            }
 
            return strOut;
        }
    }
 
    public class TextListData : InputData
    {
        List<string> m_rgstrData = new List<string>();
        List<Tuple<int[], int[]>> m_rgnData = new List<Tuple<int[], int[]>>();
        IVocabulary m_vocab;
        float[] m_rgData = null;
        float[] m_rgTgt = null;
        Phase m_phase;
        Log m_log;
 
        public enum VOCABUARY_TYPE
        {
            CHARACTER,
            WORD
        }
 
        public TextListData(Log log, string strSrcFile, string strVocabFile, bool bIncludeTarget, TokenizedDataParameter.VOCABULARY_TYPE vocabType, int? nRandomSeed = null, Phase phase = Phase.NONE) : base(nRandomSeed)
        {
            m_log = log;
            m_phase = phase;
 
            Stopwatch sw = new Stopwatch();
 
            if (vocabType == TokenizedDataParameter.VOCABULARY_TYPE.WORD)
                m_vocab = new VocabularyWord(m_random, true, true);
            else if (vocabType == TokenizedDataParameter.VOCABULARY_TYPE.SENTENCEPIECE)
                m_vocab = new VocabularySentencePiece(m_random, true, true, strVocabFile);
            else
                m_vocab = new VocabularyCharacter(m_random, true, true, true);
 
            string strProgData = Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData);
            strSrcFile = Utility.ReplaceMacro(strSrcFile, "$ProgramData$", strProgData);
 
            string[] rgstr = File.ReadAllLines(strSrcFile);
 
            sw.Start();
 
            for (int i = 0; i < rgstr.Length; i++)
            {
                m_rgstrData.Add(rgstr[i]);
                m_vocab.Add(rgstr[i]);
 
                if (sw.Elapsed.TotalMilliseconds > 1000)
                {
                    sw.Restart();
                    double dfPct = (double)i/(double)rgstr.Length;
                    m_log.Progress = dfPct;
                    m_log.WriteLine("Loading vocabulary " + i.ToString("N0") + " of " + rgstr.Length.ToString("N0") + " (" + dfPct.ToString("P") + ") ...", true);
                }
            }
 
            m_vocab.Build();
            
            for (int i = 0; i < m_rgstrData.Count; i++)
            {
                string str = m_rgstrData[i];
                int[] rgnSrc = m_vocab.Tokenize(str, bIncludeTarget, !bIncludeTarget);
                int[] rgnTrg = null;
 
                if (bIncludeTarget)
                    rgnTrg = m_vocab.CreateTarget(rgnSrc);
 
                m_rgnData.Add(new Tuple<int[], int[]>(rgnSrc, rgnTrg));
 
                if (sw.Elapsed.TotalMilliseconds > 1000)
                {
                    sw.Restart();
                    double dfPct = (double)i / (double)m_rgstrData.Count;
                    m_log.Progress = dfPct;
                    m_log.WriteLine("Tokenizing data " + i.ToString("N0") + " of " + m_rgstrData.Count.ToString("N0") + " (" + dfPct.ToString("P") + ") ...", true);
                }
            }
 
            m_log.WriteLine("'" + strSrcFile + "' vocabulary size = " + m_vocab.Count.ToString("N0"));
        }
 
        public override List<string> RawData
        {
            get { return m_rgstrData; }
        }
 
        public override uint TokenSize
        {
            get { return 1; }
        }
 
        public override uint VocabularySize
        {
            get { return (uint)m_vocab.Count; }
        }
 
        public override bool GetDataAvailabilityAt(int nIdx, bool bIncludeSrc, bool bIncludeTrg)
        {
            if (bIncludeSrc && m_rgnData[nIdx].Item1.Length == 0)
                return false;
 
            if (bIncludeTrg && m_rgnData[nIdx].Item2.Length == 0)
                return false;
 
            return true;
        }
 
        public override Tuple<float[], float[]> GetData(int nBatchSize, int nBlockSize, InputData trgData, out int[] rgnIdx)
        {
            int nSize = nBatchSize * nBlockSize;
 
            if (m_rgData == null || m_rgData.Length != nSize)
                m_rgData = new float[nSize];
            else
                Array.Clear(m_rgData, 0, m_rgData.Length);
 
            if (m_rgTgt == null || m_rgTgt.Length != nSize)
                m_rgTgt = new float[nSize];
            else
                Array.Clear(m_rgTgt, 0, m_rgTgt.Length);
 
            rgnIdx = new int[nBatchSize];
 
            for (int i = 0; i < nBatchSize; i++)
            {
                int nDataIdx = m_random.Next(m_rgnData.Count);
                int[] rgSrc = m_rgnData[nDataIdx].Item1;
                int nRetryCount = 0;
 
                while (rgSrc.Length == 0 || !trgData.GetDataAvailabilityAt(nDataIdx, true, true))
                {
                    nDataIdx = m_random.Next(m_rgnData.Count);
                    rgSrc = m_rgnData[nDataIdx].Item1;
 
                    nRetryCount++;
                    if (nRetryCount > 20 && (rgSrc.Length == 0 || !trgData.GetDataAvailabilityAt(nDataIdx, true, true)))
                        throw new Exception("Could not find a non-empty source data item!");
                }
 
                int[] rgTrg = m_rgnData[nDataIdx].Item2;
                int nDstIdx = i * nBlockSize;
 
                rgnIdx[i] = nDataIdx;
 
 
                for (int j = 0; j < nBlockSize; j++)
                {
                    if (j < rgSrc.Length)
                        m_rgData[nDstIdx + j] = rgSrc[j];
 
                    if (rgTrg != null && j < rgTrg.Length)
                        m_rgTgt[nDstIdx + j] = rgTrg[j];
                }
 
                if (rgTrg != null &&
                    rgTrg[rgTrg.Length - 1] == EOS &&
                    m_rgTgt[nDstIdx + nBlockSize - 1] != 0 &&
                    m_rgTgt[nDstIdx + nBlockSize - 1] != EOS)
                    m_rgTgt[nDstIdx + nBlockSize - 1] = EOS;
 
                if (rgSrc[rgSrc.Length - 1] == EOS &&
                    m_rgData[nDstIdx + nBlockSize - 1] != 0 &&
                    m_rgData[nDstIdx + nBlockSize - 1] != EOS)
                    m_rgData[nDstIdx + nBlockSize - 1] = EOS;
            }
 
            return new Tuple<float[], float[]>(m_rgData, m_rgTgt);
        }
 
        public override Tuple<float[], float[]> GetDataAt(int nBatchSize, int nBlockSize, int[] rgnIdx)
        {
            int nSize = nBatchSize * nBlockSize;
 
            if (m_rgData == null || m_rgData.Length != nSize)
                m_rgData = new float[nSize];
            else
                Array.Clear(m_rgData, 0, m_rgData.Length);
 
            if (m_rgTgt == null || m_rgTgt.Length != nSize)
                m_rgTgt = new float[nSize];
            else
                Array.Clear(m_rgTgt, 0, m_rgTgt.Length);
 
            for (int i = 0; i < rgnIdx.Length; i++)
            {
                int nDataIdx = rgnIdx[i];
                int nDstIdx = i * nBlockSize;
 
                int[] rgSrc = m_rgnData[nDataIdx].Item1;
                int[] rgTrg = m_rgnData[nDataIdx].Item2;
 
                for (int j = 0; j < nBlockSize; j++)
                {
                    if (j < rgSrc.Length)
                        m_rgData[nDstIdx + j] = rgSrc[j];
 
                    if (j < rgTrg.Length && rgTrg != null)
                        m_rgTgt[nDstIdx + j] = rgTrg[j];
                }
 
                if (rgTrg != null &&
                    rgTrg[rgTrg.Length - 1] == EOS &&
                    m_rgTgt[nDstIdx + nBlockSize - 1] != 0 &&
                    m_rgTgt[nDstIdx + nBlockSize - 1] != EOS)
                    m_rgTgt[nDstIdx + nBlockSize - 1] = EOS;
 
                if (rgSrc[rgSrc.Length - 1] == EOS &&
                    m_rgData[nDstIdx + nBlockSize - 1] != 0 &&
                    m_rgData[nDstIdx + nBlockSize - 1] != EOS)
                    m_rgData[nDstIdx + nBlockSize - 1] = EOS;
            }
 
            return new Tuple<float[], float[]>(m_rgData, m_rgTgt);
        }
 
        public override List<int> Tokenize(string str, bool bAddBos, bool bAddEos)
        {
            return m_vocab.Tokenize(str, bAddBos, bAddEos).ToList();
        }
 
        public override string Detokenize(float[] rgfTokIdx, int nStartIdx, int nCount, bool bIgnoreBos, bool bIgnoreEos)
        {
            string str = "";
            for (int i=nStartIdx; i<nStartIdx + nCount; i++)
            {
                string strItem = m_vocab.Detokenize((int)rgfTokIdx[i], bIgnoreBos, bIgnoreEos);
                if (string.IsNullOrEmpty(strItem))
                    break;
 
                str += strItem + " ";
            }
 
            return str.TrimEnd(' ');
        }
 
        public override string Detokenize(int nTokIdx, bool bIgnoreBos, bool bIgnoreEos)
        {
            return m_vocab.Detokenize(nTokIdx, bIgnoreBos, bIgnoreEos);
        }
 
        public override char BOS
        {
            get { return m_vocab.BOS; }
        }
 
        public override char EOS
        {
            get { return m_vocab.EOS; }
        }
    }
 
    public class CustomListData : InputData
    {
        List<Tuple<DateTime, int[], int[]>> m_rgnData = new List<Tuple<DateTime, int[], int[]>>();
        List<int> m_rgVocabulary = new List<int>();
        ICustomTokenInput m_iTokenInput;
        string m_strVocabInfo;
        float[] m_rgData = null;
        float[] m_rgTgt = null;
        Phase m_phase;
        Log m_log;
        int m_nVocabularySize;
 
        public CustomListData(CancelEvent evtCancel, Log log, string strCustomDllFile, string strVocabInfo, int nBlockSizeSrc, int? nRandomSeed = null, Phase phase = Phase.NONE) : base(nRandomSeed)
        {
            m_log = log;
            m_phase = phase;
            m_strVocabInfo = strVocabInfo;
 
            if (phase != Phase.RUN)
            {
                string strProgData = Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData);
                strCustomDllFile = Utility.ReplaceMacro(strCustomDllFile, "$ProgramData$", strProgData);
 
                m_iTokenInput = loadCustomInput(strCustomDllFile);
 
                m_rgnData = (strVocabInfo == "ENC") ? m_iTokenInput.LoadAllEncoderTokens(evtCancel, m_log, phase, out m_nVocabularySize) : m_iTokenInput.LoadAllDecoderTokens(evtCancel, m_log, phase, out m_nVocabularySize);
                if (m_rgnData.Count < nBlockSizeSrc + nBlockSizeSrc)
                    throw new Exception("Insufficient number of tokens, must have at least " + (nBlockSizeSrc + nBlockSizeSrc).ToString() + " tokens.");
 
                log.WriteLine(strVocabInfo + " vocabulary size = " + m_nVocabularySize.ToString());
            }
        }
 
        private ICustomTokenInput loadCustomInput(string strCustomDllFile)
        {
            try
            {
                Assembly a = Assembly.LoadFile(strCustomDllFile);
                AssemblyName aName = a.GetName();
 
                foreach (Type t in a.GetTypes())
                {
                    if (t.IsPublic)
                    {
                        Type iface = t.GetInterface("ICustomTokenInput");
 
                        if (iface != null)
                        {
                            object obj = Activator.CreateInstance(t);
                            return (ICustomTokenInput)obj;
                        }
                    }
                }
 
                return null;
            }
            catch (Exception excpt)
            {
                throw excpt;
            }
        }
 
        public override List<string> RawData
        {
            get { throw new NotImplementedException("Raw data not supported by Custom Input"); }
        }
 
        public override uint TokenSize
        {
            get { return 1; }
        }
 
        public override uint VocabularySize
        {
            get { return (uint)m_nVocabularySize; }
        }
 
        public override bool GetDataAvailabilityAt(int nIdx, bool bIncludeSrc, bool bIncludeTrg)
        {
            if (bIncludeSrc && m_rgnData[nIdx].Item2.Length == 0)
                return false;
 
            if (bIncludeTrg && m_rgnData[nIdx].Item3.Length == 0)
                return false;
 
            return true;
        }
 
        public override Tuple<float[], float[]> GetData(int nBatchSize, int nBlockSize, InputData trgData, out int[] rgnIdx)
        {
            int nSize = nBatchSize * nBlockSize;
 
            if (m_rgData == null || m_rgData.Length != nSize)
                m_rgData = new float[nSize];
            else
                Array.Clear(m_rgData, 0, m_rgData.Length);
 
            if (m_rgTgt == null || m_rgTgt.Length != nSize)
                m_rgTgt = new float[nSize];
            else
                Array.Clear(m_rgTgt, 0, m_rgTgt.Length);
 
            rgnIdx = new int[nBatchSize];
 
            for (int i = 0; i < nBatchSize; i++)
            {
                int nDataIdx = m_random.Next(m_rgnData.Count);
                int[] rgSrc = m_rgnData[nDataIdx].Item2;
                int nRetryCount = 0;
 
                while (rgSrc.Length == 0 || !trgData.GetDataAvailabilityAt(nDataIdx, true, true))
                {
                    nDataIdx = m_random.Next(m_rgnData.Count);
                    rgSrc = m_rgnData[nDataIdx].Item2;
 
                    nRetryCount++;
                    if (nRetryCount > 20 && (rgSrc.Length == 0 || !trgData.GetDataAvailabilityAt(nDataIdx, true, true)))
                        throw new Exception("Could not find a non-empty source data item!");
                }
 
                int[] rgTrg = m_rgnData[nDataIdx].Item3;
                int nDstIdx = i * nBlockSize;
 
                rgnIdx[i] = nDataIdx;
 
 
                for (int j = 0; j < nBlockSize; j++)
                {
                    if (j < rgSrc.Length)
                        m_rgData[nDstIdx + j] = rgSrc[j];
 
                    if (rgTrg != null && j < rgTrg.Length)
                        m_rgTgt[nDstIdx + j] = rgTrg[j];
                }
 
                if (rgTrg != null &&
                    rgTrg[rgTrg.Length - 1] == EOS &&
                    m_rgTgt[nDstIdx + nBlockSize - 1] != 0 &&
                    m_rgTgt[nDstIdx + nBlockSize - 1] != EOS)
                    m_rgTgt[nDstIdx + nBlockSize - 1] = EOS;
 
                if (rgSrc[rgSrc.Length - 1] == EOS &&
                    m_rgData[nDstIdx + nBlockSize - 1] != 0 &&
                    m_rgData[nDstIdx + nBlockSize - 1] != EOS)
                    m_rgData[nDstIdx + nBlockSize - 1] = EOS;
            }
 
            return new Tuple<float[], float[]>(m_rgData, m_rgTgt);
        }
 
        public override Tuple<float[], float[]> GetDataAt(int nBatchSize, int nBlockSize, int[] rgnIdx)
        {
            int nSize = nBatchSize * nBlockSize;
 
            if (m_rgData == null || m_rgData.Length != nSize)
                m_rgData = new float[nSize];
            else
                Array.Clear(m_rgData, 0, m_rgData.Length);
 
            if (m_rgTgt == null || m_rgTgt.Length != nSize)
                m_rgTgt = new float[nSize];
            else
                Array.Clear(m_rgTgt, 0, m_rgTgt.Length);
 
            for (int i = 0; i < rgnIdx.Length; i++)
            {
                int nDataIdx = rgnIdx[i];
                int nDstIdx = i * nBlockSize;
 
                int[] rgSrc = m_rgnData[nDataIdx].Item2;
                int[] rgTrg = m_rgnData[nDataIdx].Item3;
 
                for (int j = 0; j < nBlockSize; j++)
                {
                    if (j < rgSrc.Length)
                        m_rgData[nDstIdx + j] = rgSrc[j];
 
                    if (j < rgTrg.Length && rgTrg != null)
                        m_rgTgt[nDstIdx + j] = rgTrg[j];
                }
 
                if (rgTrg != null &&
                    rgTrg[rgTrg.Length - 1] == EOS &&
                    m_rgTgt[nDstIdx + nBlockSize - 1] != 0 &&
                    m_rgTgt[nDstIdx + nBlockSize - 1] != EOS)
                    m_rgTgt[nDstIdx + nBlockSize - 1] = EOS;
 
                if (rgSrc[rgSrc.Length - 1] == EOS &&
                    m_rgData[nDstIdx + nBlockSize - 1] != 0 &&
                    m_rgData[nDstIdx + nBlockSize - 1] != EOS)
                    m_rgData[nDstIdx + nBlockSize - 1] = EOS;
            }
 
            return new Tuple<float[], float[]>(m_rgData, m_rgTgt);
        }
 
        public override List<int> Tokenize(string str, bool bAddBos, bool bAddEos)
        {
            throw new NotImplementedException("Tokenize not supported by Custom Input.");
        }
 
        public override string Detokenize(float[] rgfTokIdx, int nStartIdx, int nCount, bool bIgnoreBos, bool bIgnoreEos)
        {
            throw new NotImplementedException("Detokenize not supported by Custom Input.");
        }
 
        public override string Detokenize(int nTokIdx, bool bIgnoreBos, bool bIgnoreEos)
        {
            throw new NotImplementedException("Detokenize not supported by Custom Input.");
        }
 
        public override char BOS
        {
            get { return (char)SPECIAL_TOKENS.BOS; }
        }
 
        public override char EOS
        {
            get { return (char)SPECIAL_TOKENS.EOS; }
        }
    }
}