data_accessor_splitting_mem.hpp

Go to the documentation of this file.

#ifndef FSTDATAACCESSORSPLITTINGMEM_H
#define FSTDATAACCESSORSPLITTINGMEM_H
 /* 
=========================================================================
Copyright and license:
  * FST4 software (with the exception of any bundled or externally linked
    third-party components) is Copyright (C) 2000-present Institute of
    Information Theory and Automation of the Czech Academy of Sciences (UTIA).
  * FST4 is free software: you can redistribute it and/or modify it under
    the terms of the GNU Affero General Public License as published by the
    Free Software Foundation, version 3. The full text of the license is in
    the file LICENSE in the distribution root and at
    https://www.gnu.org/licenses/agpl-3.0.html.
  * FST4 is distributed in the hope that it will be useful, but WITHOUT ANY
    WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
    License for more details.
  * Closed-source use without the AGPL's copyleft obligations requires a
    commercial license from UTIA; in case of interest write to
    utia@utia.cas.cz. LICENSING.md in the distribution root explains both
    options, the contributor license agreement that every contribution
    requires, and the contacts (technical questions and pull requests:
    somol@utia.cas.cz, with "FST" in the subject line).
========================================================================== */
 
#include <memory>
#include <cassert>
#include <iostream>
#include <sstream>
#include <vector>
 
#include "error.hpp"
#include "global.hpp"
#include "data_accessor_splitting.hpp"
#include "data_scaler.hpp"
 
/*============== Template parameter type naming conventions ==============
--------- Numeric types: -------------------------------------------------
DATATYPE - data sample values - usually real numbers (but may be integers
          in text processing etc.)
REALTYPE - must be real numbers - for representing intermediate results of 
          calculations like mean, covariance etc.
IDXTYPE - index values for enumeration of data samples - (nonnegative) integers, 
          extent depends on numbers of samples in data
DIMTYPE - index values for enumeration of features (dimensions), or classes (not 
          class sizes) - (nonnegative) integers, usually lower extent than IDXTYPE, 
          but be aware of expressions like _classes*_features*_features ! 
          in linearized representations of feature matrices for all classes
BINTYPE - feature selection marker type - represents ca. <10 different feature 
          states (selected, deselected, sel./desel. temporarily 1st nested loop, 2nd...)
RETURNTYPE - criterion value: real value, but may be extended in future to support 
          multiple values 
--------- Class types: ---------------------------------------------------
SUBSET       - class of class type Subset 
CLASSIFIER   - class implementing interface defined in abstract class Classifier 
EVALUATOR    - class implementing interface defined in abstract class Sequential_Step 
DISTANCE     - class implementing interface defined in abstract class Distance 
DATAACCESSOR - class implementing interface defined in abstract class Data_Accessor 
INTERVALCONTAINER - class of class type Data_Intervaller 
CONTAINER    - STL container of class type Data_Interval  
========================================================================== */
 
namespace FST {

template<typename DATATYPE, typename IDXTYPE, class INTERVALCONTAINER>
class Data_Accessor_Splitting_Mem : public Data_Accessor_Splitting<DATATYPE,IDXTYPE,INTERVALCONTAINER>
{
        // NOTE: the calling convention is the same for the inner and the outer CV loop -- both call just
        //   get*Split, get*Block, get*Pattern -- and which of them a call refers to is selected beforehand
        //   with the inherited setSplittingDepth()
        public:
                typedef Data_Accessor_Splitting<DATATYPE,IDXTYPE,INTERVALCONTAINER> DAS; 
                typedef std::shared_ptr<Data_Scaler<DATATYPE> > PScaler; 
                typedef typename DAS::PPattern PPattern; 
                typedef typename DAS::PSplitters PSplitters; 
                Data_Accessor_Splitting_Mem(const std::string _filename, const PSplitters _dsp, const PScaler _dsc) : Data_Accessor_Splitting<DATATYPE,IDXTYPE,INTERVALCONTAINER>(_dsp), dsc(_dsc), filename(_filename), datasize(0), scaler_fit_train(false) {notify("Data_Accessor_Splitting_Mem constructor.");}
                virtual ~Data_Accessor_Splitting_Mem(){notify("Data_Accessor_Splitting_Mem destructor.");}

                virtual void initialize();

                void set_scaler_fit_train(const bool b) {scaler_fit_train=b;}
                bool get_scaler_fit_train() const {return scaler_fit_train;} 

                void scale_sample_inplace(DATATYPE *sample) const;
 
                virtual bool getFirstSplit() override;
                virtual bool getNextSplit() override;
 
                virtual bool getFirstBlock(const DataPart ofwhat, PPattern &firstpattern, IDXTYPE &patterns, const unsigned int loopdepth=0) override;
                virtual bool getNextBlock(const DataPart ofwhat, PPattern &firstpattern, IDXTYPE &patterns, const unsigned int loopdepth=0) override;
 
                Data_Accessor_Splitting_Mem* clone() const override {throw fst_error("Data_Accessor_Splitting_Mem::clone() not supported, use Data_Accessor_Splitting_Mem::sharing_clone() instead.");}
                Data_Accessor_Splitting_Mem* sharing_clone() const override = 0;
                Data_Accessor_Splitting_Mem* stateless_clone() const override {throw fst_error("Data_Accessor_Splitting_Mem::stateless_clone() not supported, use Data_Accessor_Splitting_Mem::sharing_clone() instead.");}
                
                virtual std::ostream& print(std::ostream& os) const override;
        protected:
                Data_Accessor_Splitting_Mem(const Data_Accessor_Splitting_Mem &damt, int);
        protected:
                // Contract for a derived (file-format) class. initialize() calls these in this order:
                // initial_file_prepare(), then the three file_get*() queries, then initial_data_read().
                // A hook that cannot do its job must throw fst_error; returning silently would leave the
                // accessor half-built. None of them may be reached on a sharing_clone.
                virtual void initial_data_read() = 0;
                virtual void initial_file_prepare() = 0;
                virtual unsigned int file_getNoOfClasses() const = 0; 
                virtual unsigned int file_getNoOfFeatures() const = 0; 
                virtual IDXTYPE file_getClassSize(unsigned int cls) const = 0; 
 
                void initial_data_scaling(); 

                void per_split_data_scaling();
 
                PScaler dsc; 
                std::vector<PScaler> feature_scalers;
 
                std::string filename; 
 
                IDXTYPE datasize; 
                std::shared_ptr<DATATYPE> data; 
                std::shared_ptr<DATATYPE> data_raw;
                bool scaler_fit_train;
                std::vector<IDXTYPE> _class_offset; 
};
 
// learns the scaler transform over the whole data set, per feature, and applies it in place;
// off-limits in shared_clone (it mutates the shared data buffer)
template<typename DATATYPE, typename IDXTYPE, class INTERVALCONTAINER>
void Data_Accessor_Splitting_Mem<DATATYPE,IDXTYPE,INTERVALCONTAINER>::initial_data_scaling()
{
        if(Clonable::is_sharing_clone()) throw fst_error("Data_Accessor_Splitting_Mem()::initial_data_scaling() called from shared_clone instance.");
        // SCALING over values per-feature, regardless classes
        assert(dsc);
        {std::ostringstream sos; sos << std::endl << "Scaling data (per-feature, over all classes) using " << *dsc << "..."; syncout::print(std::cout,sos);}
        IDXTYPE idx=0;
        IDXTYPE _features=file_getNoOfFeatures();
        feature_scalers.clear(); feature_scalers.reserve(_features);
        bool keep_transforms=true;
        for(IDXTYPE f=0;f<_features;f++) {
                for(bool b=dsc->startFirstLoop(); b==true; b=dsc->startNextLoop())
                {
                        idx=f;
                        for(unsigned int c=0;c<file_getNoOfClasses();c++) for(IDXTYPE p=0;p<file_getClassSize(c);p++)
                        {
                                dsc->learn(data.get()[idx]);
                                idx+=_features;
                        }
                }
                idx=f;
                for(unsigned int c=0;c<file_getNoOfClasses();c++) for(IDXTYPE p=0;p<file_getClassSize(c);p++)
                {
                        dsc->scale_inplace(data.get()[idx]);
                        idx+=_features;
                }
                if(keep_transforms) { // keep this feature's learned transform for scale_sample_inplace()
                        try {feature_scalers.push_back(PScaler(dsc->duplicate()));}
                        catch(fst_error&) {keep_transforms=false; feature_scalers.clear();} // a scaler without duplicate(): user vectors cannot be transformed, the data itself is unaffected
                }
        }
        {std::ostringstream sos; sos << "done." << std::endl << std::endl; syncout::print(std::cout,sos);}
}
 
template<typename DATATYPE, typename IDXTYPE, class INTERVALCONTAINER>
void Data_Accessor_Splitting_Mem<DATATYPE,IDXTYPE,INTERVALCONTAINER>::per_split_data_scaling()
{
        // scaler_fit_train mode: learn the transform on the current depth-0 TRAIN part only,
        // then re-derive the whole working buffer from the pristine data (leakage-free scaling;
        // deliberately silent — this runs on every depth-0 split change).
        // NOTE: fitting reads the physical splits[0][*]._train intervals (absolute at depth 0),
        //       so DataPart substitution (substitute()) does not affect what the scaler learns on
        assert(scaler_fit_train);
        assert(dsc);
        assert(data && data_raw);
        const unsigned int _features=DAS::features;
        const unsigned int _classes=(unsigned int)DAS::classes.size();
        feature_scalers.clear(); feature_scalers.reserve(_features);
        bool keep_transforms=true;
        for(unsigned int f=0;f<_features;f++)
        {
                for(bool b=dsc->startFirstLoop(); b==true; b=dsc->startNextLoop())
                {
                        for(unsigned int c=0;c<_classes;c++)
                        {
                                const INTERVALCONTAINER &train=*(DAS::splits[0][c]._train);
                                for(typename INTERVALCONTAINER::const_iterator iter=train.begin();iter!=train.end();++iter)
                                {
                                        IDXTYPE idx=_class_offset[c]+((IDXTYPE)iter->startidx)*_features+f;
                                        for(IDXTYPE p=0;p<(IDXTYPE)iter->count;p++,idx+=_features) dsc->learn(data_raw.get()[idx]);
                                }
                        }
                }
                // apply to ALL patterns (train and test parts alike), always from the pristine values
                IDXTYPE idx=f;
                for(unsigned int c=0;c<_classes;c++) for(IDXTYPE p=0;p<(IDXTYPE)DAS::classes[c];p++,idx+=_features)
                        data.get()[idx]=dsc->scale(data_raw.get()[idx]);
                if(keep_transforms) { // this split's transform of feature f, for scale_sample_inplace()
                        try {feature_scalers.push_back(PScaler(dsc->duplicate()));}
                        catch(fst_error&) {keep_transforms=false; feature_scalers.clear();}
                }
        }
}
 
template<typename DATATYPE, typename IDXTYPE, class INTERVALCONTAINER>
void Data_Accessor_Splitting_Mem<DATATYPE,IDXTYPE,INTERVALCONTAINER>::scale_sample_inplace(DATATYPE *sample) const
{
        assert(sample);
        if(feature_scalers.size()!=(std::size_t)DAS::features)
                throw fst_error("Data_Accessor_Splitting_Mem::scale_sample_inplace(): no per-feature scaler transform available (initialize() not called yet, or the scaler does not implement duplicate())");
        for(std::size_t f=0;f<feature_scalers.size();f++) feature_scalers[f]->scale_inplace(sample[f]);
}
 
template<typename DATATYPE, typename IDXTYPE, class INTERVALCONTAINER>
bool Data_Accessor_Splitting_Mem<DATATYPE,IDXTYPE,INTERVALCONTAINER>::getFirstSplit()
{
        const bool result=DAS::getFirstSplit();
        if(result && scaler_fit_train && DAS::getSplittingDepth()==0) per_split_data_scaling();
        return result;
}
 
template<typename DATATYPE, typename IDXTYPE, class INTERVALCONTAINER>
bool Data_Accessor_Splitting_Mem<DATATYPE,IDXTYPE,INTERVALCONTAINER>::getNextSplit()
{
        const bool result=DAS::getNextSplit();
        if(result && scaler_fit_train && DAS::getSplittingDepth()==0) per_split_data_scaling();
        return result;
}
 
template<typename DATATYPE, typename IDXTYPE, class INTERVALCONTAINER>
void Data_Accessor_Splitting_Mem<DATATYPE,IDXTYPE,INTERVALCONTAINER>::initialize()
{
        // NOTE: throws I/O and memory related exceptions; off-limits in shared_clone
        if(Clonable::is_sharing_clone()) throw fst_error("Data_Accessor_Splitting_Mem()::initialize() called from shared_clone instance.");
        
        assert(DAS::dsp->size()>0);
        try {
                {std::ostringstream sos; sos << "Opening file: " << filename << std::endl; syncout::print(std::cout,sos);}
                initial_file_prepare();
                {
                        std::ostringstream sos;
                        sos << "Features: " << file_getNoOfFeatures() << std::endl << "Classes: " << file_getNoOfClasses() << std::endl;
                        syncout::print(std::cout,sos);
                }
                        
                typename DAS::CLASSSIZES csizes;
                for(unsigned int c=0;c<file_getNoOfClasses();c++) 
                {
                        {std::ostringstream sos; sos << "  Class "<<c<<" size: "<<file_getClassSize(c) << std::endl; syncout::print(std::cout,sos);} // 0-based, like every class index FST4 prints or accepts (--f1-positive, --class-weights, confusion rows) and like the feature indices
                        csizes.push_back(file_getClassSize(c));
                }
                {
                        std::ostringstream sos;
                        sos << "Splitting depth: " << DAS::dsp->size() << std::endl;
                        syncout::print(std::cout,sos);
                }
                
                DAS::initialize(file_getNoOfFeatures(), csizes); // set-up memory structures to support data splitting
                
                datasize=0;
                _class_offset.clear();
                for(unsigned int c=0;c<file_getNoOfClasses();c++) {
                        _class_offset.push_back(datasize); // to be set only once
                        datasize+=file_getClassSize(c)*file_getNoOfFeatures();
                }
 
                data.reset(new DATATYPE[datasize], std::default_delete<DATATYPE[]>());
                initial_data_read();
                if(!scaler_fit_train) initial_data_scaling();
                else {
                        // per-split scaling: keep the loaded data pristine (shared, read-only), work on a private
                        // copy; the transform is learned per depth-0 split in per_split_data_scaling() — before the
                        // first split is made no transform is defined, so pre-split access sees unscaled data
                        assert(dsc);
                        data_raw=data;
                        data.reset(new DATATYPE[datasize], std::default_delete<DATATYPE[]>());
                        std::copy(data_raw.get(),data_raw.get()+datasize,data.get());
                        {std::ostringstream sos; sos << std::endl << "Per-split scaling enabled (fit on each depth-0 TRAIN part) using " << *dsc << "." << std::endl; syncout::print(std::cout,sos);}
                }
 
        } catch (...) {
                notify("Data_Accessor_Splitting_Mem::initialize() failed.");
                throw;
        }
}
 
template<typename DATATYPE, typename IDXTYPE, class INTERVALCONTAINER>
bool Data_Accessor_Splitting_Mem<DATATYPE,IDXTYPE,INTERVALCONTAINER>::getFirstBlock(const DataPart ofwhat, PPattern &firstpattern, IDXTYPE &patterns, const unsigned int loopdepth)
{
        assert(data);
        typename DAS::DATAINTERVAL tmp;
        if(!DAS::getFirstBlock(ofwhat,tmp,loopdepth)) return false;
        firstpattern=&data.get()[_class_offset[DAS::active_class]+tmp->startidx*DAS::features];
        patterns=tmp->count;
        return true;
}
 
template<typename DATATYPE, typename IDXTYPE, class INTERVALCONTAINER>
bool Data_Accessor_Splitting_Mem<DATATYPE,IDXTYPE,INTERVALCONTAINER>::getNextBlock(const DataPart ofwhat, PPattern &firstpattern, IDXTYPE &patterns, const unsigned int loopdepth)
{
        assert(data);
        typename DAS::DATAINTERVAL tmp;
        if(!DAS::getNextBlock(ofwhat,tmp,loopdepth)) return false;
        firstpattern=&data.get()[_class_offset[DAS::active_class]+tmp->startidx*DAS::features];
        patterns=tmp->count;
        return true;
}
 
template<typename DATATYPE, typename IDXTYPE, class INTERVALCONTAINER>
Data_Accessor_Splitting_Mem<DATATYPE,IDXTYPE,INTERVALCONTAINER>::Data_Accessor_Splitting_Mem(const Data_Accessor_Splitting_Mem& damt, int) : Data_Accessor_Splitting<DATATYPE,IDXTYPE,INTERVALCONTAINER>(damt),
        feature_scalers(damt.feature_scalers),
        datasize(damt.datasize),
        data(damt.data),
        data_raw(damt.data_raw),
        scaler_fit_train(damt.scaler_fit_train),
        _class_offset(damt._class_offset)
{
        if(scaler_fit_train) {
                // per-split scaling: the clone re-learns the transform whenever it advances depth-0 splits,
                // so it needs a private working buffer (initialized to the parent's current state — correct
                // for any access that does not advance splits) and a private scaler instance
                if(!damt.dsc) throw fst_error("Data_Accessor_Splitting_Mem weak cc: scaler_fit_train mode requires a scaler instance in the source.");
                data.reset(new DATATYPE[datasize], std::default_delete<DATATYPE[]>());
                std::copy(damt.data.get(),damt.data.get()+datasize,data.get());
                dsc.reset(damt.dsc->duplicate());
        } else dsc.reset();
        filename.clear();
        notify("Data_Accessor_Splitting_Mem weak copy-constructor.");
}
 
template<typename DATATYPE, typename IDXTYPE, class INTERVALCONTAINER>
std::ostream& Data_Accessor_Splitting_Mem<DATATYPE,IDXTYPE,INTERVALCONTAINER>::print(std::ostream& os) const 
{
        DAS::print(os); 
        os << std::endl << "Data_Accessor_Splitting_Mem(file=" << filename << ")";
        return os;
}
        
} // namespace
#endif // FSTDATAACCESSORSPLITTINGMEM_H ///:~