DataSchema.cs

using MyCaffe.basecode;
using MyCaffe.basecode.descriptors;
using MyCaffe.db.temporal;
using System;
using System.Collections;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Xml;
 
namespace MyCaffe.layers.tft
{
    public class DataSchema
    {
        Data m_data = new Data();
        LookupCollection m_lookups = new LookupCollection();
 
        public DataSchema()
        {
        }
 
        public static DataSchema LoadFromDatasetDescriptor(DatasetDescriptor desc, Phase phase)
        {
            DataSchema schema = new DataSchema();
 
            if (desc.TestingSource == null && desc.TrainingSource == null)
            {
                if (desc.ID == 0)
                    throw new Exception("The dataset '" + desc.Name + "' has not been saved to the database and cannot be loaded.  The data layer should be configured to use a database dataset, but may be configured to use an .NPY file as a source.");
 
                DatabaseLoader loader = new DatabaseLoader();
                int nDsID = desc.ID;
                desc = loader.LoadDatasetFromDb(nDsID);
            }
 
            TemporalDescriptor tsd = (phase == Phase.TEST) ? desc.TestingSource.TemporalDescriptor : desc.TrainingSource.TemporalDescriptor;
 
            if (tsd == null)
                return null;
 
            schema.Data.Columns = tsd.ValueStreamDescriptors[0].Steps;
 
            foreach (ValueStreamDescriptor vsd in tsd.ValueStreamDescriptors)
            {
                switch (vsd.ClassType)
                {
                    case ValueStreamDescriptor.STREAM_CLASS_TYPE.STATIC:
                        if (vsd.ValueType == ValueStreamDescriptor.STREAM_VALUE_TYPE.NUMERIC)
                            schema.Data.StaticNum.Add(new Field(vsd));
                        else
                            schema.Data.StaticCat.Add(new Field(vsd));
                        break;
 
                    case ValueStreamDescriptor.STREAM_CLASS_TYPE.OBSERVED:
                        if (vsd.ValueType == ValueStreamDescriptor.STREAM_VALUE_TYPE.NUMERIC)
                            schema.Data.ObservedNum.Add(new Field(vsd));
                        else
                            schema.Data.ObservedCat.Add(new Field(vsd));
                        break;
 
                    case ValueStreamDescriptor.STREAM_CLASS_TYPE.KNOWN:
                        if (vsd.ValueType == ValueStreamDescriptor.STREAM_VALUE_TYPE.NUMERIC)
                            schema.Data.KnownNum.Add(new Field(vsd));
                        else
                            schema.Data.KnownCat.Add(new Field(vsd));
                        break;
                }
            }
 
            return schema;
        }
 
        public static DataSchema LoadFromProject(ProjectEx prj, Phase phase = Phase.TRAIN)
        {
            return LoadFromDatasetDescriptor(prj.Dataset, phase);
        }
 
        public static DataSchema Load(string strFile)
        {
            DataSchema schema = new DataSchema();
 
            XmlDocument xmlDoc = new XmlDocument();
            xmlDoc.Load(strFile);
 
            XmlNode node = xmlDoc.SelectSingleNode("Schema");
            if (node == null)
                return null;
 
            XmlNode dataNode = node.SelectSingleNode("Data");
            XmlAttribute colAtt = dataNode.Attributes["Columns"];
            schema.Data.Columns = int.Parse(colAtt.Value);
 
            XmlNode syncNode = dataNode.SelectSingleNode("Sync");
            schema.m_data.LoadSync(syncNode);
 
            XmlNode observedNode = dataNode.SelectSingleNode("Observed");
            XmlNode observedNumNode = observedNode.SelectSingleNode("Numeric");
            schema.m_data.LoadObservedNum(observedNumNode);
            XmlNode observedCatNode = observedNode.SelectSingleNode("Categorical");
            schema.m_data.LoadObservedCat(observedCatNode);
 
            XmlNode knownNode = dataNode.SelectSingleNode("//Known");
            XmlNode knownNumNode = knownNode.SelectSingleNode("Numeric");
            schema.m_data.LoadKnownNum(knownNumNode);
            XmlNode knownCatNode = knownNode.SelectSingleNode("Categorical");
            schema.m_data.LoadKnownCat(knownCatNode);
 
            XmlNode staticNode = dataNode.SelectSingleNode("Static");
            XmlNode staticNumNode = staticNode.SelectSingleNode("Numeric");
            schema.m_data.LoadStaticNum(staticNumNode);
            XmlNode staticCatNode = staticNode.SelectSingleNode("Categorical");
            schema.m_data.LoadStaticCat(staticCatNode);
 
            XmlNode lookupsNode = node.SelectSingleNode("Lookups");
            schema.m_lookups = LookupCollection.Load(lookupsNode);
 
            return schema;
        }
 
        public Data Data
        {
            get { return m_data; }
        }
 
        public LookupCollection Lookups
        {
            get { return m_lookups; }
        }
    }
 
    public class Data
    {
        int m_nColumns = 0;
        FieldCollection m_sync = new FieldCollection();
        FieldCollection m_numKnown = new FieldCollection();
        FieldCollection m_catKnown = new FieldCollection();
        FieldCollection m_numObserved = new FieldCollection();
        FieldCollection m_catObserved = new FieldCollection();
        FieldCollection m_numStatic = new FieldCollection();
        FieldCollection m_catStatic = new FieldCollection();
 
        public Data()
        {
        }
 
        public void LoadSync(XmlNode node)
        {
            m_sync = FieldCollection.Load(node);
        }
 
        public void LoadKnownNum(XmlNode node)
        {
            m_numKnown = FieldCollection.Load(node);
        }
 
        public void LoadKnownCat(XmlNode node)
        {
            m_catKnown = FieldCollection.Load(node);
        }
 
        public void LoadObservedNum(XmlNode node)
        {
            m_numObserved = FieldCollection.Load(node);
        }
 
        public void LoadObservedCat(XmlNode node)
        {
            m_catObserved = FieldCollection.Load(node);
        }
 
        public void LoadStaticNum(XmlNode node)
        {
            m_numStatic = FieldCollection.Load(node);
        }
 
        public void LoadStaticCat(XmlNode node)
        {
            m_catStatic = FieldCollection.Load(node);
        }
 
        public int TargetIndex
        {
            get { return m_numObserved.FindFieldIndex(Field.INPUT_TYPE.TARGET); }
        }
 
        public int Columns
        {
            get { return m_nColumns; }
            set { m_nColumns = value; }
        }
 
        public FieldCollection Sync
        {
            get { return m_sync; }
        }
 
        public FieldCollection KnownNum
        {
            get { return m_numKnown; }
        }
 
        public FieldCollection KnownCat
        {
            get { return m_catKnown; }
        }
 
        public FieldCollection ObservedNum
        {
            get { return m_numObserved; }
        }
 
        public int ObservedNumExplicitCount
        {
            get
            {
                int nCount = 0;
                foreach (Field field in m_numObserved)
                {
                    if ((field.InputType & Field.INPUT_TYPE.OBSERVED) == Field.INPUT_TYPE.OBSERVED)
                        nCount++;
                }
 
                return nCount;
            }
        }
 
        public bool IsObservedNum(int nIdx)
        {
            if (nIdx < 0 || nIdx >= m_numObserved.Count)
                return false;
 
            return ((m_numObserved[nIdx].InputType & Field.INPUT_TYPE.OBSERVED) == Field.INPUT_TYPE.OBSERVED);
        }
 
        public FieldCollection ObservedCat
        {
            get { return m_catObserved; }
        }
 
        public FieldCollection StaticNum
        {
            get { return m_numStatic; }
        }
 
        public FieldCollection StaticCat
        {
            get { return m_catStatic; }
        }
    }
 
    public class LookupCollection
    {
        Dictionary<string, Lookup> m_rgLookups = new Dictionary<string, Lookup>();
 
        public LookupCollection()
        {
        }
 
        public static LookupCollection Load(XmlNode node)
        {
            LookupCollection lookups = new LookupCollection();
 
            XmlNodeList lookupNodes = node.SelectNodes("Lookup");
            foreach (XmlNode lookupNode in lookupNodes)
            {
                Lookup lookup = Lookup.Load(lookupNode);
                lookups.Add(lookup);
            }
 
            return lookups;
        }
 
        public void Add(Lookup lookup)
        {
            m_rgLookups.Add(lookup.Name, lookup);
        }
 
        public Lookup Find(string strName)
        {
            if (!m_rgLookups.ContainsKey(strName))
                return null;
 
            return m_rgLookups[strName];
        }
 
        public Lookup this[int nIdx]
        {
            get { return m_rgLookups.ElementAt(nIdx).Value; }
        }
 
        public int Count
        {
            get { return m_rgLookups.Count; }
        }
    }
 
    public class Lookup
    {
        string m_strName;
        Dictionary<string, LookupItem> m_rgLookupNameToId = new Dictionary<string, LookupItem>();
        Dictionary<int, LookupItem> m_rgLookupIdToName = new Dictionary<int, LookupItem>();
 
        public Lookup()
        {
        }
 
        public static Lookup Load(XmlNode node)
        {
            Lookup lookup = new Lookup();
 
            lookup.m_strName = node.Attributes["Name"].Value;
 
            XmlNodeList itemNodes = node.SelectNodes("Item");
            foreach (XmlNode itemNode in itemNodes)
            {
                lookup.Add(LookupItem.Load(itemNode));
            }
 
            return lookup;
        }
 
        public int Count
        {
            get { return m_rgLookupIdToName.Count; }
        }
 
        public void Clear()
        {
            m_rgLookupIdToName.Clear();
            m_rgLookupNameToId.Clear();
        }
 
        public void Add(Lookup l)
        {
            foreach (KeyValuePair<int, LookupItem> kv in l.m_rgLookupIdToName)
            {
                bool bAdd = true;
 
                if (m_rgLookupIdToName.ContainsKey(kv.Key))
                {
                    if (!m_rgLookupIdToName[kv.Key].Compare(kv.Value))
                    {
                        m_rgLookupIdToName.Remove(kv.Key);
                        m_rgLookupNameToId.Remove(kv.Value.Name);
                    }
                    else
                    {
                        bAdd = false;
                    }
                }
 
                if (bAdd)
                {
                    m_rgLookupIdToName.Add(kv.Key, kv.Value);
                    m_rgLookupNameToId.Add(kv.Value.Name, kv.Value);
                }
            }
        }
 
        public void Add(LookupItem item)
        {
            m_rgLookupIdToName.Add(item.ID, item);
            m_rgLookupNameToId.Add(item.Name, item);
        }
 
        public string Name
        {
            get { return m_strName; }
        }
 
        public int FindID(string strName)
        {
            if (!m_rgLookupNameToId.ContainsKey(strName))
                return -1;
 
            return m_rgLookupNameToId[strName].ID;
        }
 
        public string FindName(int nID)
        {
            if (!m_rgLookupIdToName.ContainsKey(nID))
                return null;
 
            return m_rgLookupIdToName[nID].Name;
        }
 
        public LookupItem this[int nIdx]
        {
            get { return m_rgLookupIdToName.ElementAt(nIdx).Value; }
        }
    }
 
    public class LookupItem
    {
        string m_strName;
        int m_nIndex;
        int m_nValidRangeStartIndex = -1;
        int m_nValidRangeEndIndex = -1;
 
        public LookupItem(string strName, int nIndex)
        {
            m_strName = strName;
            m_nIndex = nIndex;
        }
 
        public LookupItem(string strName, int nIndex, int nValidRangeStart, int nValidRangeEnd)
        {
            m_strName = strName;
            m_nIndex = nIndex;
            m_nValidRangeStartIndex = nValidRangeStart;
            m_nValidRangeEndIndex = nValidRangeEnd;
        }
 
        public static LookupItem Load(XmlNode node)
        {
            string strName = node.FirstChild.Value;
            int nIndex = int.Parse(node.Attributes["Index"].Value);
            int nValidRangeStart = -1;
            int nValidRangeEnd = -1;
 
            if (node.Attributes["ValidRangeStartIdx"] != null)
                nValidRangeStart = int.Parse(node.Attributes["ValidRangeStartIdx"].Value);
 
            if (node.Attributes["ValidRangeEndIdx"] != null)
                nValidRangeEnd = int.Parse(node.Attributes["ValidRangeEndIdx"].Value);
 
            return new LookupItem(strName, nIndex, nValidRangeStart, nValidRangeEnd);
        }
 
        public bool Compare(LookupItem item)
        {
            if (item.Name != Name)
                return false;
 
            if (item.ValidRangeEndIndex != ValidRangeEndIndex)
                return false;
 
            if (item.ValidRangeStartIndex != ValidRangeStartIndex)
                return false;
 
            return true;
        }
 
        public string Name
        {
            get { return m_strName; }
        }
 
        public int ID
        {
            get { return m_nIndex; }
        }
 
        public int ValidRangeStartIndex
        {
            get { return m_nValidRangeStartIndex; }
        }
 
        public int ValidRangeEndIndex
        {
            get { return m_nValidRangeEndIndex; }
        }
 
        public int ValidRangeCount
        {
            get { return (m_nValidRangeStartIndex == -1) ? -1 : (m_nValidRangeEndIndex - m_nValidRangeStartIndex) + 1; }
        }
 
        public override string ToString()
        {
            return m_strName + " @" + m_nIndex.ToString() + " (" + m_nValidRangeStartIndex.ToString() + "," + m_nValidRangeEndIndex.ToString() + ")";
        }
    }
 
    public class FieldCollection : IEnumerable<Field>
    {
        string m_strFile;
        List<Field> m_rgFields = new List<Field>();
 
        public FieldCollection()
        {
        }
 
        public static FieldCollection Load(XmlNode node)
        {
            FieldCollection col = new FieldCollection();
 
            XmlNode nodeFile = node.SelectSingleNode("File");
            if (nodeFile != null)
                col.m_strFile = nodeFile.FirstChild.Value;
 
            XmlNodeList nodes = node.SelectNodes("Field");
            foreach (XmlNode node1 in nodes)
            {
                Field field = Field.Load(node1);
                col.Add(field);
            }
 
            return col;
        }
 
        public int FindFieldIndex(Field.INPUT_TYPE type)
        {
            for (int i = 0; i < m_rgFields.Count; i++)
            {
                if ((m_rgFields[i].InputType & type) == type)
                    return i;
            }
 
            return -1;
        }
 
        public void Add(Field field)
        {
            m_rgFields.Add(field);
        }
 
        public IEnumerator<Field> GetEnumerator()
        {
            return m_rgFields.GetEnumerator();
        }
 
        IEnumerator IEnumerable.GetEnumerator()
        {
            return m_rgFields.GetEnumerator();
        }
 
        public Field this[int nIdx]
        {
            get { return m_rgFields[nIdx]; }
        }
 
        public Field this[string strName]
        {
            get
            {
                foreach (Field field in m_rgFields)
                {
                    if (field.Name == strName)
                        return field;
                }
 
                return null;
            }
        }
 
        public int Count
        {
            get { return m_rgFields.Count; }
        }
 
        public string File
        {
            get { return m_strFile; }
        }
    }
 
    public class Field
    {
        string m_strName;
        int m_nIdx = 0;
        DATA_TYPE m_dataType = DATA_TYPE.REAL;
        INPUT_TYPE m_inputType = INPUT_TYPE.TIME;
 
        public enum DATA_TYPE
        {
            REAL,
            CATEGORICAL
        }
 
        public enum INPUT_TYPE
        {
            NONE = 0x0000,
            TIME = 0x0001,
            ID = 0x0002,
            KNOWN = 0x0004,
            OBSERVED = 0x0008,
            STATIC = 0x0010,
            TARGET = 0x1000,
        }
 
        public Field(string strName, int nIdx, DATA_TYPE dataType, INPUT_TYPE inputType)
        {
            m_strName = strName;
            m_nIdx = nIdx;
            m_dataType = dataType;
            m_inputType = inputType;
        }
 
        public Field(ValueStreamDescriptor vsd)
        {
            m_strName = vsd.Name;
            m_nIdx = vsd.Ordering;
            m_dataType = (vsd.ValueType == ValueStreamDescriptor.STREAM_VALUE_TYPE.NUMERIC) ? DATA_TYPE.REAL : DATA_TYPE.CATEGORICAL;
 
            switch (vsd.ClassType)
            {
                case ValueStreamDescriptor.STREAM_CLASS_TYPE.STATIC:
                    m_inputType = INPUT_TYPE.STATIC;
                    break;
 
                case ValueStreamDescriptor.STREAM_CLASS_TYPE.OBSERVED:
                    m_inputType = INPUT_TYPE.OBSERVED;
                    break;
 
                case ValueStreamDescriptor.STREAM_CLASS_TYPE.KNOWN:
                    m_inputType = INPUT_TYPE.KNOWN;
                    break;
            }
        }
 
        public static Field Load(XmlNode node)
        {
            string strName = node.FirstChild.Value;
            int nIdx = int.Parse(node.Attributes["Index"].Value);
            string strDataType = node.Attributes["DataType"].Value;
            string strInputType = node.Attributes["InputType"].Value;
 
            DATA_TYPE dataType = DATA_TYPE.REAL;
            INPUT_TYPE inputType = INPUT_TYPE.NONE;
 
            if (strDataType == "REAL")
                dataType = DATA_TYPE.REAL;
            else if (strDataType == "CATEGORICAL")
                dataType = DATA_TYPE.CATEGORICAL;
 
            if (strInputType == "TIME")
                inputType = INPUT_TYPE.TIME;
            else if (strInputType == "ID")
                inputType = INPUT_TYPE.ID;
            else if (strInputType == "KNOWN")
                inputType = INPUT_TYPE.KNOWN;
            else if (strInputType == "STATIC")
                inputType = INPUT_TYPE.STATIC;
            else
            {
                if (strInputType.Contains("OBSERVED"))
                    inputType |= INPUT_TYPE.OBSERVED;
                if (strInputType.Contains("TARGET"))
                    inputType |= INPUT_TYPE.TARGET;
            }
 
            return new Field(strName, nIdx, dataType, inputType);
        }
 
        public string Name
        {
            get { return m_strName; }
        }
 
        public int Index
        {
            get { return m_nIdx; }
        }
 
        public DATA_TYPE DataType
        {
            get { return m_dataType; }
        }
 
        public INPUT_TYPE InputType
        {
            get { return m_inputType; }
        }
 
        override public string ToString()
        {
            return m_strName + " @" + m_nIdx.ToString() + " (" + m_dataType.ToString() + "," + m_inputType.ToString() + ")";
        }
    }
}