data_file_ARFF.hpp

Go to the documentation of this file.

#ifndef FSTDATAFILEARFF_H
#define FSTDATAFILEARFF_H
 /* 
=========================================================================
Copyright and license:
  * FST4 software (with the exception of any bundled or externally linked
    third-party components) is Copyright (C) 2000-present Institute of
    Information Theory and Automation of the Czech Academy of Sciences (UTIA).
  * FST4 is free software: you can redistribute it and/or modify it under
    the terms of the GNU Affero General Public License as published by the
    Free Software Foundation, version 3. The full text of the license is in
    the file LICENSE in the distribution root and at
    https://www.gnu.org/licenses/agpl-3.0.html.
  * FST4 is distributed in the hope that it will be useful, but WITHOUT ANY
    WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
    License for more details.
  * Closed-source use without the AGPL's copyleft obligations requires a
    commercial license from UTIA; in case of interest write to
    utia@utia.cas.cz. LICENSING.md in the distribution root explains both
    options, the contributor license agreement that every contribution
    requires, and the contacts (technical questions and pull requests:
    somol@utia.cas.cz, with "FST" in the subject line).
========================================================================== */
 
#include <cstdio>
#include <cstring>
#include <cctype>
#include <cstdlib>
#include <vector>
#include <iostream>
#include "error.hpp"
#include "global.hpp"
 
namespace FST {

 

        class Data_File_ARFF_Feature {
        public:
                static const int TYPE_NUMERIC = 1;      
                static const int TYPE_NOMINAL = 2;      
                static const int TYPE_STRING = 3;       
                static const int TYPE_UNSUPPORTED = 4;  
 
                std::string name;                  
                std::string typeName;           
                int type;                          
                int offset;                        
                int size;                          
                bool isClass;                      
                std::vector<std::string> values;                        
        };

        class Data_File_ARFF_Record {
                void *data;    
                int classId;   
 
        public:
                Data_File_ARFF_Record(int recordSize) {this->data = malloc(recordSize); classId = -1;} 
                ~Data_File_ARFF_Record();
 
                void *getData() { return this->data;} 
                int getClassId() { return this->classId;} 
                void setClassId(int pId) {this->classId = pId;} 
        };
 
        inline Data_File_ARFF_Record::~Data_File_ARFF_Record() {
                if (this->data != nullptr) free(this->data);
                this->data = nullptr;
        }
 
 


        class Data_File_ARFF {
        private:
                std::string _fileName;  
                std::vector<Data_File_ARFF_Record*> data;  
                std::vector<Data_File_ARFF_Feature*> attributes;        
                std::vector<Data_File_ARFF_Feature*> features;  
                //  graceful degradation over unsupported column types: columnMap has one entry per
                //  @ATTRIBUTE line in file order — the index into `attributes` for loaded columns,
                //  -1 for skipped ones. Row parsers walk file columns through this map, so a
                //  STRING/DATE/unknown-typed column does not abort the load; each skip is reported
                //  loudly and the load summary states that feature indices follow the REMAINING
                //  columns in file order.
                struct SkippedColumn {
                        std::string name;      
                        std::string typeName;  
                        int fileColumn;        
                };
                std::vector<SkippedColumn> skippedColumns;  
                std::vector<int> columnMap;  
                int *classSize;  
 
                std::string relationName;  
                int attributesCount;  
                int classAttribute;  
                int classCount;  
 
                int recordSize;  
                bool loaded;  
        public:
                Data_File_ARFF(std::string fileName);  
                ~Data_File_ARFF();

                bool prepareFile();                             // prepares the file contents

                void sortRecordsByClass();

                unsigned int getNoOfClasses();                  // resturn number of classes
                unsigned int getNoOfFeatures() {return features.size();} 
                unsigned int getNoOfAttributes() {return attributes.size();} 
                unsigned int getTotalSize();                            // returns the total number of records
                unsigned int getClassSize(int clsId);   // returns number of records for given class
                std::string getClassName(int clsId); // returns the nam of given class
 
                Data_File_ARFF_Record* getRecordAt(long i) {if (i<0 || i>= (long)data.size()) return nullptr; else return data[i];} 
                //  returns the value of given feature of given record
                float getFeatureValue(long record, int feature) {
                        char *v = ((char *)(data[record]->getData()))+features[feature]->offset;
                        if (features[feature]->type == Data_File_ARFF_Feature::TYPE_NOMINAL) return (float)(*(short *)v); // nominal values are stored as short (see readAttrValue)
                        return *(float *)v;
                };
 
                void printInfo(); 
        private:
                Data_File_ARFF_Feature *createAttribute(char *info);  
                bool readNextRow(char *row);
                void prepareAttributes();
                Data_File_ARFF_Record *readNormalRow(char *row);
                Data_File_ARFF_Record *readSparseRow(char *row);
                bool readAttrValue(Data_File_ARFF_Feature *feature, Data_File_ARFF_Record *rec, char *value);

                bool readLine( FILE *f, std::vector<char> &buff ) {
                        size_t len = 0; buff[0] = 0;
                        while (fgets(buff.data()+len, (int)(buff.size()-len), f) != nullptr) {
                                len += strlen(buff.data()+len);
                                if ((len > 0 && buff[len-1] == '\n') || feof(f)) return true;
                                if (len >= buff.size()-2) buff.resize(2*buff.size());
                        }
                        return len > 0;
                };

                char *trim( char *szSource ) {
                        char *pszEOS = szSource + strlen( szSource ) - 1;
                        while( pszEOS >= szSource && (*pszEOS == ' ' || *pszEOS == 9 || *pszEOS == 13 || *pszEOS == 10)) *pszEOS-- = '\0';
                        pszEOS = szSource;
                        while( *pszEOS == ' ' || *pszEOS == 9  || *pszEOS == 13 || *pszEOS == 10) pszEOS++;
                        return pszEOS;
                };

                char *mystrupr( char *szSource ) {
                        char *pszEOS = szSource;
                        while( *pszEOS != 0 ) {*pszEOS=toupper(*pszEOS); pszEOS++;}
                        return szSource;
                };
        };
 
 
        inline Data_File_ARFF::Data_File_ARFF(std::string fileName) {
                this->_fileName = fileName;
                this->classSize = nullptr;
                attributesCount = 0;
                classAttribute = -1;
                classCount = -1;
                loaded = false;
        };
 
        inline Data_File_ARFF::~Data_File_ARFF() {
                for (unsigned i=0;i<attributes.size(); i++) delete attributes[i];
                this->attributes.clear();
                this->features.clear();
                for (unsigned i=0;i<data.size(); i++) delete data[i];
                this->data.clear();
                if (classSize != nullptr) delete[] classSize;
                classSize = nullptr;
        }
 
        inline bool Data_File_ARFF::prepareFile() {
                FILE *f =fopen(_fileName.c_str(),"rt");
                if(f==nullptr) {throw FST::fst_error("Data_File_ARFF file read problem.",1);}
                std::vector<char> buff(8192);   //  grown by readLine() as needed — no line-length cap
 
                bool readingData = false;
                bool isError = false;
                int rowsReaded = 0;
 
                while (!isError && readLine(f, buff)) {
                        // NOTE: trim() right-trims in place but LEFT-trims only through its return value,
                        // so the result must be used — reading buff.data() instead would leave an indented
                        // @DATA/@ATTRIBUTE line unrecognized
                        char *line = trim(buff.data());
 
                        if (!line[0]) continue;         //  empty row
                        if (line[0]=='%') continue;                     // comment
                        if (readingData) {
                                if (readNextRow(line)) rowsReaded++;
                                else isError = true;
                        } else if (line[0]=='@') {
                                //  get the second part of string
                                char *val = line+2; while(*val!=' ' && *val!=0 && *val!=9) val++; val[0] = 0; val++;
                                trim(mystrupr(line));
 
                                if (strcmp(line, "@RELATION")==0) this->relationName = val;
                                else if (strcmp(line, "@DATA")==0) {
                                        readingData = true;
                                        prepareAttributes();
                                } else if (strcmp(line, "@ATTRIBUTE")==0) {
                                        Data_File_ARFF_Feature *feature = this->createAttribute(val);
                                        if (feature == nullptr) {       //  malformed definition — a damaged header stays fatal
                                                std::cout << "Attribute description not understood : " << line << " " << val << std::endl;
                                                isError = true; break;
                                        } else if (feature->type == Data_File_ARFF_Feature::TYPE_STRING ||
                                                   feature->type == Data_File_ARFF_Feature::TYPE_UNSUPPORTED) {
                                                //  unsupported type: skip the COLUMN, keep loading everything else
                                                const int fileColumn = (int)columnMap.size();
                                                std::cout << "Skipping attribute '" << feature->name.c_str() << "' (file column "
                                                          << fileColumn+1 << "): unsupported type " << feature->typeName.c_str()
                                                          << " - FST4 features are numeric/nominal" << std::endl;
                                                skippedColumns.push_back(SkippedColumn{feature->name, feature->typeName, fileColumn});
                                                columnMap.push_back(-1);
                                                delete feature;
                                        } else {                                        //  loadable feature
                                                columnMap.push_back((int)attributes.size());
                                                this->attributes.push_back(feature);
                                                this->attributesCount ++;
                                        }
                                }
                        } else {
                                //  a line outside the header that is neither a comment nor data: reported,
                                //  but tolerated — the load continues with the next line
                                if (line[0] != 0) {
                                        std::cout << "Error reading file - unexpected row : " << line << std::endl;
                                }
                        }
 
                }
 
                fclose(f);
 
                //  graceful degradation ends where nothing usable remains
                if (!isError && !skippedColumns.empty() && attributes.empty()) {
                        std::cout << "No loadable attribute columns remain (all " << columnMap.size()
                                  << " have unsupported types)" << std::endl;
                        throw FST::fst_error("Data_File_ARFF: no loadable attributes (all column types unsupported).",1);
                }
 
                loaded = !isError;
                if (!loaded) return false; // the offending definition/row was reported above; the summary
                                           // below reads through getters that refuse an unloaded file
                std::cout <<  "Relation: " << this->relationName.c_str() << std::endl;
                std::cout <<  "Classes count: " << getNoOfClasses() << std::endl;
                std::cout <<  "Records count: " << rowsReaded << std::endl;
                if (!skippedColumns.empty()) {
                        std::cout << "Skipped " << skippedColumns.size() << " of " << columnMap.size()
                                  << " attribute columns (unsupported types, reported above); loaded feature"
                                  << " indices follow the remaining columns in file order" << std::endl;
                }
 
                return !isError;
        }
 
        inline void Data_File_ARFF::printInfo() {
                if (!loaded) {std::cout << "File not loaded" << std::endl; return;};
                for (unsigned d=0; d < data.size(); d++) {
                        char *dt = (char *)data[d]->getData();
                        std::cout << "Record: cls=" << data[d]->getClassId() << " " ;
                        for (unsigned i=0; i < attributes.size(); i++) {
                                char *v = dt + attributes[i]->offset;
                                if (attributes[i]->type == Data_File_ARFF_Feature::TYPE_NOMINAL) 
                                        std::cout << " " << *(short *)v;
                                if (attributes[i]->type == Data_File_ARFF_Feature::TYPE_NUMERIC) 
                                        std::cout << " " << *(float*)v;
                        }
                        std::cout << std::endl;
                }
 
                std::cout <<  "Relation: " << this->relationName.c_str() << std::endl;
                std::cout <<  "Feature count: " << this->attributes.size() << std::endl;
 
                for (unsigned i=0; i < attributes.size(); i++) {
                        std::cout << "  " << attributes[i]->name.c_str() << " : " << attributes[i]->type << std::endl;
                        if (attributes[i]->type == Data_File_ARFF_Feature::TYPE_NOMINAL) {
                                std::vector<std::string> vals = attributes[i]->values;
                                for (unsigned j=0; j < vals.size(); j++) 
                                        std::cout << "    - " << vals[j].c_str() << std::endl;
                        }
                }
 
                std::cout <<  "Classes count: " << getNoOfClasses() << " sizes: ";
                for (unsigned int i=0; i<getNoOfClasses() ; i++)
                        std::cout << getClassName(i).c_str() << ": " << getClassSize(i) << "     ";
                std::cout << std::endl;
 
 
                std::cout <<  "Records count: " << data.size() << std::endl;
 
        }
 
        inline void Data_File_ARFF::prepareAttributes() {
                int size = 0;
                this->classAttribute = -1;
 
                for (unsigned fi = 0; fi < attributes.size(); fi ++ ) {
                        Data_File_ARFF_Feature *feature = attributes[fi];
                        feature->offset = size;
                        size += feature->size;
 
                        std::vector<char> tmp(feature->name.c_str(), feature->name.c_str()+feature->name.length()+1); // no VLA: MSVC hard error C2131
                        mystrupr(tmp.data());
                        if ((strcmp(tmp.data(), "CLASS")==0 || strcmp(tmp.data(), "'CLASS'")==0) && feature->type==Data_File_ARFF_Feature::TYPE_NOMINAL) {
                                this->classAttribute = fi;
                                feature->isClass = true;
                        } else {
                                feature->isClass = false;
                                features.push_back(feature);
                        }
                }
 
                if (classAttribute>=0) {
                        classCount = attributes[classAttribute]->values.size();
                        classSize = new int[classCount];
                        for (int i=0;i<classCount;i++) classSize[i] = 0;
                        std::cout << "Classes count : " << classCount << std::endl;
                }
                this->recordSize = size;
        }
 
        inline Data_File_ARFF_Feature *Data_File_ARFF::createAttribute(char *info) {
                info = trim(info);
                if (*info == 0) {std::cout << "empty definition!" << std::endl;return nullptr;}
 
                //  get the second part: the name ends at the first blank -- unless it is quoted ('my attr'
                //  or "my attr", which is how Weka writes names containing blanks), in which case it ends
                //  at the closing quote and the quotes are not part of the name
                char *val;
                if (*info == '\'' || *info == '"') {
                        const char qc = *info; info++;
                        val = info; while (*val != qc && *val != 0) val++;
                        if (*val == 0) {std::cout << "syntax error (unterminated quoted attribute name)! - " << info << std::endl; return nullptr;}
                        *val = 0; val++;
                } else {
                        val = info+1; while(*val!=' ' && *val!=0 && *val!=9) val++;
                        if (*val) {*val = 0; val++;}
                }
                val = trim(val);
                // info => name
                // val => type
 
                int type = 0;
                int size = 0;
 
                Data_File_ARFF_Feature *feature = nullptr;
 
                if (val[0]==0)  {std::cout << "empty datatype!" << std::endl;return nullptr;}
                if (val[0]=='{') {              
                        val++;
                        char *p = val;
                        std::vector<std::string> values;
                        while (*val != '}' && *val !=0) {
                                if (*val == ',') {
                                        *val = 0;
                                        std::vector<char> tmp(p, p+strlen(p)+1); // no VLA: MSVC hard error C2131
                                        values.push_back((std::string)trim(tmp.data()));
                                        p = ++val;
                                }
                                val++;
                        }
                        if (*val == 0) {std::cout << "syntax error! - " << info << std::endl;return nullptr;}
                        *val = 0;
                        std::vector<char> tmp(p, p+strlen(p)+1); // no VLA: MSVC hard error C2131
                        values.push_back((std::string)trim(tmp.data()));
 
                        feature = new Data_File_ARFF_Feature();
                        feature->name = info;
                        feature->size = sizeof(short);  //  short
                        feature->type = Data_File_ARFF_Feature::TYPE_NOMINAL;
                        feature->values = values;
 
                } else {
                        val = mystrupr(val);
                        if (strcmp(val, "REAL")==0 ||  strcmp(val, "NUMERIC")==0 || strcmp(val, "INTEGER")==0) {
                                type = Data_File_ARFF_Feature::TYPE_NUMERIC;
                                size = sizeof(float);
                        } else if (strcmp(val, "STRING")==0) {
                                // an unsupported-type MARKER, not a loadable feature: FST4 has no way to turn
                                // free text into a numeric feature, so the caller skips the whole column
                                type = Data_File_ARFF_Feature::TYPE_STRING;
                                size = 0;
                        } else {
                                // unknown/unsupported type (DATE, RELATIONAL, typos...) — marker, same contract
                                type = Data_File_ARFF_Feature::TYPE_UNSUPPORTED;
                                size = 0;
                        }
                        feature = new Data_File_ARFF_Feature();
                        feature->name = info;
                        feature->typeName = val;
                        feature->size = size;
                        feature->type = type;
                }
                return feature;
        }
 
        inline bool Data_File_ARFF::readNextRow(char *row) {
                if (! *row) return false;               //  empty row;
                row = trim(row);
                Data_File_ARFF_Record *rec = nullptr;
                if (*row == '{') { 
                        rec = this->readSparseRow(row);
                } else {
                        rec = this->readNormalRow(row);
                }
 
                if (rec == nullptr) return false;
                
                if (classSize != nullptr && rec->getClassId()>=0) classSize[rec->getClassId()] ++;
                this->data.push_back(rec);
                return true;
        }
 
        inline bool Data_File_ARFF::readAttrValue(Data_File_ARFF_Feature *feature, Data_File_ARFF_Record *rec, char *value) {
                //  p points to single value for my feature
                char *fdata = (char *)rec->getData()+feature->offset;
                if (feature->type == Data_File_ARFF_Feature::TYPE_NUMERIC) {
                        if (sscanf(value, "%f", (float *)fdata)!=1) {
                                std::cout << "Error reading feature data - feature " << feature->name << ":"<< value << std::endl;return false;
                        }
                        if (feature->isClass) rec->setClassId((int)*((float*)fdata)); // slot holds a float (see sscanf above)
                } else if (feature->type == Data_File_ARFF_Feature::TYPE_NOMINAL) {
                        std::vector<std::string> vals = feature->values;
                        int val = -1;
                        for (unsigned j=0; j < vals.size(); j++) 
                                if (vals[j].compare(value)==0) {val = j; break;};
                        if (val == -1) {
                                std::cout << "Nominal value not found - " << feature->name.c_str() << " " << value << std::endl; return false;
                        };
                        *(short *)(fdata) = (short)val;
 
                        if (feature->isClass) rec->setClassId(val);
                } else {
                        std::cout << "Unknown attribute type - " << feature->name.c_str() << feature->type << std::endl; return false;
                }
                return true;
        }
 
        inline Data_File_ARFF_Record *Data_File_ARFF::readNormalRow(char *row) {
                Data_File_ARFF_Record *rec = new Data_File_ARFF_Record(recordSize);
                bool ok = true;
 
                //  iterate FILE columns; columnMap routes each token to its loaded
                //  attribute or discards it (skipped unsupported column)
                for (unsigned ci = 0; ok && ci < columnMap.size(); ci ++ ) {
                        if (*row == 0) {        //  problem, row ended to soon
                                std::cout << "Error reading data record - column " << ci << std::endl; ok = false; break;
                        }
                        //  quote-aware token scan: honor a leading ' or " so commas embedded in
                        //  quoted values (typical for skipped STRING columns) don't split the token
                        char *p = row;
                        char *q = row; while (*q == ' ' || *q == 9) q++;
                        if (*q == '\'' || *q == '"') {const char qc = *q; q++; while (*q != 0 && *q != qc) q++; if (*q) q++; row = q;}
                        while (*row != 0 && *row != ',') row++;
                        char tmpch = *row; *row = 0;
 
                        const int ai = columnMap[ci];
                        if (ai >= 0) {
                                // tokens must be trimmed before nominal lookup — a space after a comma
                                // would otherwise fail the exact string compare against the declared values
                                char *tok = trim(p);
                                // a value may be single- or double-quoted (Weka quotes nominal values freely,
                                // and quoted numbers occur too): hand readAttrValue() the bare content
                                { const size_t L = strlen(tok); if (L >= 2 && (tok[0] == '\'' || tok[0] == '"') && tok[L-1] == tok[0]) {tok[L-1] = 0; tok = trim(tok+1);} }
                                ok = ok && this->readAttrValue(attributes[ai], rec, tok);
                        }       //  ai<0: unsupported column — token consumed and discarded
 
                        *row = tmpch;
                        if (*row == ',') row++;
                }
 
                if (! ok ) {delete rec; return nullptr;};
                return rec;
        }
 
        inline Data_File_ARFF_Record *Data_File_ARFF::readSparseRow(char *row) {
 
                Data_File_ARFF_Record *rec = new Data_File_ARFF_Record(recordSize);
                void *data = rec->getData();
 
                memset(data, 0, recordSize);
                bool ok = true;
 
                char *id;
                char *val;
                int attID = 0;
 
                row++;
                while (*row != 0 && *row != '}' && ok) {
                        //  skip initial spaces
                        while (*row == 9 || *row == ' ') row++;
                        // start reading attrnum
                        id = row;       
                        while (*row != 9 && *row != ' ' && *row!=0) row++;
                        if (*row == 0) {ok= false; break;};     //  line can't end here
                        *row = 0; row++;
                        // => id contains attr id
 
                        //  skip spaces
                        while (*row == 9 || *row == ' ') row++;
                        val = row;
                        if (*row == '"' || *row == '\'') { //  value is in commas
                                row++;
                                while (*row != 0 && *row != *val) row++; // scan to the matching closing quote (*val is the opening one)
                                if (*row == 0) {ok= false; break;};     //  line can't end here
 
                                *row = 0; val++; row++;
                                while (*row == 9 || *row == ' ' || *row == ',') row++;
                        } else {
                                while (*row != ',' && *row != '}' && *row != 0) row++;
                                if (*row == 0) {ok= false; break;};     //  line can't end here
                                if (*row == '}') {*row=0; row++; *row = '}';}
                                else {*row = 0; row++;};
                                val = trim(val);
                        }
                        // => val contains attr value
                        //        row contains either ',' or '}'
 
                        attID = atoi(id);       //  a FILE column index — remap through columnMap
                        if (attID<0 || attID >= (int)columnMap.size()) {
                                std::cout << "Invalid attribute index - " << attID << std::endl; ok = false; break;
                        }
 
                        const int ai = columnMap[attID];
                        if (ai >= 0) {
                                ok = ok && readAttrValue(attributes[ai], rec, val);
                        }       //  ai<0: unsupported column — value ignored (reported once at header time)
 
                        while (*row == 9 || *row == ' ') row++;
                }
 
                if (!ok) {delete rec; return nullptr;}
 
                // in sparse format an omitted attribute means value 0; if the class attribute
                // was omitted the record belongs to class 0 (its data slot is already zeroed
                // by the memset above)
                if (classAttribute >= 0 && rec->getClassId() == -1) rec->setClassId(0);
 
                return rec;
        }
 
        inline unsigned int Data_File_ARFF::getNoOfClasses() {
                if (!loaded) {throw fst_error("File not loaded");};
 
                if (classAttribute == -1) {std::cout << "class attribute not found" << std::endl; return 0;};
                if (attributes[classAttribute]->type != Data_File_ARFF_Feature::TYPE_NOMINAL) {
                        // defensive only: prepareAttributes() never assigns classAttribute to a
                        // non-nominal attribute, so this state is unreachable through load();
                        // throw rather than return -1, which would wrap to a huge unsigned value
                        throw fst_error("Class attribute is not nominal");
                }
                return attributes[classAttribute]->values.size();
        }
 
        inline unsigned int Data_File_ARFF::getTotalSize() {
                if (!loaded) {throw fst_error("File not loaded");};
                return data.size();
        }
 
        inline unsigned int Data_File_ARFF::getClassSize(int clsId) {
                if (!loaded) {throw fst_error("File not loaded");};
                // throw rather than return -1, which would wrap to a huge unsigned value;
                // all in-tree callers iterate over [0,getNoOfClasses()) so never trigger this
                if (classSize == nullptr) {throw fst_error("getClassSize: no class attribute loaded");};
                if (clsId < 0 || clsId >= classCount) {throw fst_error("getClassSize: class id out of range");};
                return classSize[clsId];
        }
 
        inline std::string Data_File_ARFF::getClassName(int clsId) {
                if (!loaded) {throw fst_error("File not loaded");};
                if (classSize == nullptr || clsId < 0 || clsId >= classCount) { return "-n/a-";};
                return attributes[classAttribute]->values[clsId];
        }
 
 
        inline void Data_File_ARFF::sortRecordsByClass() {
                if (!loaded) {throw fst_error("File not loaded");};
                if (classCount < 1) {throw fst_error("Classes not identified");};
                std::vector<Data_File_ARFF_Record*> **clsData = new std::vector<Data_File_ARFF_Record*>*[classCount];
 
                for (int c=0;c<classCount;c++) {
                        clsData[c]= new std::vector<Data_File_ARFF_Record*>;
                }
 
                for (unsigned i=0;i<data.size();i++) {
                        Data_File_ARFF_Record *rec = data[i];
                        clsData[rec->getClassId()]->push_back(rec);
                }
 
                data.clear();
                for (int c=0;c<classCount;c++) {
                        std::vector<Data_File_ARFF_Record*> *v =clsData[c];
                        for (unsigned d=0;d<v->size();d++) {
                                Data_File_ARFF_Record *rec = v->at(d);
                                data.push_back(rec);
                        }
                        v->clear();
                        delete v;
                }
 
                delete[] clsData;
        }
 
}
 
#endif