Interfaces.cs

using MyCaffe.basecode;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
 
namespace MyCaffe.layers.gpt
{
    public interface IVocabulary
    {
        int Count { get; }
        char BOS { get; }
        char EOS { get; }
        void Add(string str);
        int Build();
        int BuildFromString(string strData);
        int[] CreateTarget(int[] rgSrc);
        int[] Tokenize(string str, bool bAddBos, bool bAddEos);
        List<int> Tokenize(string str1, bool bMustExist = true);
        string Detokenize(float[] rgf, bool bIgnoreBos, bool bIgnoreEos);
        string Detokenize(int nIdxToken, bool bIgnoreBos, bool bIgnoreEos);
    }
 
    public interface ICustomTokenInput
    {
        List<Tuple<DateTime, int[], int[]>> LoadAllEncoderTokens(CancelEvent evtCancel, Log log, Phase phase, out int nVocabSize);
        List<Tuple<DateTime, int[], int[]>> LoadAllDecoderTokens(CancelEvent evtCancel, Log log, Phase phase, out int nVocabSize);
    }
 
    public abstract class InputData
    {
        protected Random m_random;
 
        public InputData(int? nRandomSeed = null)
        {
            if (nRandomSeed.HasValue)
                m_random = new Random(nRandomSeed.Value);
            else
                m_random = new Random();
        }
 
        public abstract List<string> RawData { get; }
        public abstract uint TokenSize { get; }
        public abstract uint VocabularySize { get; }
        public abstract bool GetDataAvailabilityAt(int nIdx, bool bIncludeSrc, bool bIncludeTrg);
        public abstract Tuple<float[], float[]> GetData(int nBatchSize, int nBlockSize, InputData trgData, out int[] rgnIdx);
        public abstract Tuple<float[], float[]> GetDataAt(int nBatchSize, int nBlockSize, int[] rgnIdx);
        public abstract List<int> Tokenize(string str, bool bAddBos, bool bAddEos);
        public abstract string Detokenize(int nTokIdx, bool bIgnoreBos, bool bIgnoreEos);
        public abstract string Detokenize(float[] rgf, int nStartIdx, int nCount, bool bIgnoreBos, bool bIgnoreEos);
        public abstract char BOS { get; }
        public abstract char EOS { get; }
    }
}