result_weighter_sampled.hpp

Go to the documentation of this file.

#ifndef FSTRESULTWEIGHTERSAMPLED_H
#define FSTRESULTWEIGHTERSAMPLED_H
 /*
=========================================================================
Copyright and license:
  * FST4 software (with the exception of any bundled or externally linked
    third-party components) is Copyright (C) 2000-present Institute of
    Information Theory and Automation of the Czech Academy of Sciences (UTIA).
  * FST4 is free software: you can redistribute it and/or modify it under
    the terms of the GNU Affero General Public License as published by the
    Free Software Foundation, version 3. The full text of the license is in
    the file LICENSE in the distribution root and at
    https://www.gnu.org/licenses/agpl-3.0.html.
  * FST4 is distributed in the hope that it will be useful, but WITHOUT ANY
    WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
    License for more details.
  * Closed-source use without the AGPL's copyleft obligations requires a
    commercial license from UTIA; in case of interest write to
    utia@utia.cas.cz. LICENSING.md in the distribution root explains both
    options, the contributor license agreement that every contribution
    requires, and the contacts (technical questions and pull requests:
    somol@utia.cas.cz, with "FST" in the subject line).
========================================================================== */
 
#include <memory>
#include <vector>
#include <mutex>
#include <cmath>
#include <cassert>
#include <sstream>
#include <algorithm>
#include "error.hpp"
#include "global.hpp"
 
/*============== Template parameter type naming conventions ==============
--------- Numeric types: -------------------------------------------------
DATATYPE - data sample values - usually real numbers (but may be integers
          in text processing etc.)
REALTYPE - must be real numbers - for representing intermediate results of
          calculations like mean, covariance etc.
IDXTYPE - index values for enumeration of data samples - (nonnegative) integers,
          extent depends on numbers of samples in data
DIMTYPE - index values for enumeration of features (dimensions), or classes (not
          class sizes) - (nonnegative) integers, usually lower extent than IDXTYPE,
          but be aware of expressions like _classes*_features*_features !
          in linearized representations of feature matrices for all classes
BINTYPE - feature selection marker type - represents ca. <10 different feature
          states (selected, deselected, sel./desel. temporarily 1st nested loop, 2nd...)
RETURNTYPE - criterion value: real value, but may be extended in future to support
          multiple values
--------- Class types: ---------------------------------------------------
SUBSET       - class of class type Subset
CRITERION    - class implementing interface defined in abstract class Criterion
========================================================================== */
 
namespace FST {

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>
class Result_Weighter_Sampled {
public:
        typedef std::shared_ptr<SUBSET> PSubset; 

        enum StatsMode { STATS_ZCONTRAST, STATS_RAWMEAN };

        Result_Weighter_Sampled(const IDXTYPE horizon=100, const StatsMode mode=STATS_ZCONTRAST)
                : _n(0), _horizon(horizon<1?1:horizon), _mode(mode), _frozen(false), _total_evals(0), _flushed_batches(0)
                {notify("Result_Weighter_Sampled constructor.");}
        virtual ~Result_Weighter_Sampled() {notify("Result_Weighter_Sampled destructor.");}

        void reset(const DIMTYPE n);
        DIMTYPE get_n() const {return _n;} 
        StatsMode get_mode() const {return _mode;} 
        IDXTYPE get_horizon() const {return _horizon;} 

        bool add(const RETURNTYPE value, const PSubset sub);
        IDXTYPE flush_batch(); 

        void freeze() {flush_batch(); std::lock_guard<std::mutex> lock(_add_mutex); _frozen=true;}
        bool is_frozen() const {return _frozen;} 
 
        RETURNTYPE score(const DIMTYPE f) const; 
        IDXTYPE count_is(const DIMTYPE f) const {assert(f<_n); return _fs[f].n_is;} 
        IDXTYPE count_isnot(const DIMTYPE f) const {assert(f<_n); return _fs[f].n_isnot;} 
        IDXTYPE total_evals() const {return _total_evals;} 
        IDXTYPE flushed_batches() const {return _flushed_batches;} 

        std::ostream& print(std::ostream& os) const {os << "Result_Weighter_Sampled(n=" << _n << ", horizon=" << _horizon << ", stats=" << (_mode==STATS_ZCONTRAST?"zcontrast":"rawmean") << (_frozen?", FROZEN":"") << ", evals=" << _total_evals << ")"; return os;}
        bool print_stats(std::ostream& os, const DIMTYPE top=20) const; 
 
protected:
        struct FeatureStat {
                RETURNTYPE m_is;        
                RETURNTYPE m_isnot;     
                RETURNTYPE v_is;        
                RETURNTYPE raw_sum_is;  
                RETURNTYPE raw_cnt_is;  
                IDXTYPE n_is;    
                IDXTYPE n_isnot; 
        };

        struct BatchEntry {
                RETURNTYPE value;              
                DIMTYPE d;                     
                std::vector<DIMTYPE> selected; 
        };
        DIMTYPE _n; 
        IDXTYPE _horizon; 
        StatsMode _mode; 
        bool _frozen; 
        std::vector<FeatureStat> _fs; 
        std::vector<BatchEntry> _batch; 
        std::vector<char> _mark;               
        IDXTYPE _total_evals;     
        IDXTYPE _flushed_batches; 
        std::mutex _add_mutex; 
};
 
template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>
void Result_Weighter_Sampled<RETURNTYPE,IDXTYPE,DIMTYPE,SUBSET>::reset(const DIMTYPE n)
{
        assert(n>0);
        _n=n;
        _fs.assign(_n,FeatureStat());
        for(DIMTYPE f=0;f<_n;f++) {
                _fs[f].m_is=0.0; _fs[f].m_isnot=0.0; _fs[f].v_is=0.0;
                _fs[f].raw_sum_is=0.0; _fs[f].raw_cnt_is=0.0;
                _fs[f].n_is=0; _fs[f].n_isnot=0;
        }
        _mark.assign(_n,0);
        _batch.clear();
        _frozen=false;
        _total_evals=0;
        _flushed_batches=0;
}
 
template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>
bool Result_Weighter_Sampled<RETURNTYPE,IDXTYPE,DIMTYPE,SUBSET>::add(const RETURNTYPE value, const PSubset sub)
{
        assert(sub);
        std::lock_guard<std::mutex> lock(_add_mutex);
        if(_frozen) return true; // frozen ranking: the no-op is a success, callers must not treat it as an error
        if(_n==0) reset(sub->get_n());
        assert(_n==sub->get_n());
        if(_n!=sub->get_n()) return false;
        BatchEntry e;
        e.value=value;
        e.d=sub->get_d();
        e.selected.reserve(e.d);
        for(DIMTYPE f : sub->selected_features()) e.selected.push_back(f);
        _batch.push_back(std::move(e));
        _total_evals++;
        return true;
}
 
template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>
IDXTYPE Result_Weighter_Sampled<RETURNTYPE,IDXTYPE,DIMTYPE,SUBSET>::flush_batch()
{
        std::lock_guard<std::mutex> lock(_add_mutex);
        if(_frozen) {_batch.clear(); return 0;}
        const IDXTYPE B=(IDXTYPE)_batch.size();
        if(B==0) return 0;
        assert(_n>0);
        double mu=0.0;
        for(IDXTYPE i=0;i<B;i++) mu+=(double)_batch[i].value;
        mu/=(double)B;
        double var=0.0;
        for(IDXTYPE i=0;i<B;i++) {const double dv=(double)_batch[i].value-mu; var+=dv*dv;}
        var/=(double)B;
        const double sd=std::sqrt(var);
        const bool degenerate=!(sd>1e-12); // constant batch (or B==1) carries no ranking information -> z=0
        for(IDXTYPE i=0;i<B;i++) {
                const BatchEntry &e=_batch[i];
                assert(e.d==_batch[0].d); // caller contract: one batch = one cardinality
                const RETURNTYPE z=degenerate?(RETURNTYPE)0.0:(RETURNTYPE)(((double)e.value-mu)/sd);
                for(size_t k=0;k<e.selected.size();k++) _mark[e.selected[k]]=1;
                const RETURNTYPE dinv=(e.d>0)?(RETURNTYPE)(1.0/(double)e.d):(RETURNTYPE)0.0;
                for(DIMTYPE j=0;j<_n;j++) {
                        FeatureStat &s=_fs[j];
                        if(_mark[j]) {
                                s.n_is++;
                                const RETURNTYPE a=(RETURNTYPE)1.0/(RETURNTYPE)(s.n_is<_horizon?s.n_is:_horizon);
                                s.m_is+=a*(z-s.m_is);
                                s.v_is+=a*(z*z-s.v_is);
                                s.raw_sum_is+=dinv*e.value;
                                s.raw_cnt_is+=dinv;
                        } else {
                                s.n_isnot++;
                                const RETURNTYPE a=(RETURNTYPE)1.0/(RETURNTYPE)(s.n_isnot<_horizon?s.n_isnot:_horizon);
                                s.m_isnot+=a*(z-s.m_isnot);
                        }
                }
                for(size_t k=0;k<e.selected.size();k++) _mark[e.selected[k]]=0;
        }
        _batch.clear();
        _flushed_batches++;
        return B;
}
 
template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>
RETURNTYPE Result_Weighter_Sampled<RETURNTYPE,IDXTYPE,DIMTYPE,SUBSET>::score(const DIMTYPE f) const
{
        assert(f<_n);
        const FeatureStat &s=_fs[f];
        if(_mode==STATS_RAWMEAN) return (s.raw_cnt_is>0.0)?(s.raw_sum_is/s.raw_cnt_is):(RETURNTYPE)0.0;
        return s.m_is-s.m_isnot;
}
 
template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>
bool Result_Weighter_Sampled<RETURNTYPE,IDXTYPE,DIMTYPE,SUBSET>::print_stats(std::ostream& os, const DIMTYPE top) const
{
        std::ostringstream sos;
        if(_n<1) {sos << "Result_Weighter_Sampled: nothing to print." << std::endl; syncout::print(os,sos); return false;}
        std::vector<DIMTYPE> idx(_n);
        for(DIMTYPE f=0;f<_n;f++) idx[f]=f;
        const DIMTYPE k=(top>0 && top<_n)?top:_n;
        std::partial_sort(idx.begin(),idx.begin()+k,idx.end(),
                [this](const DIMTYPE a,const DIMTYPE b){const RETURNTYPE sa=score(a),sb=score(b); return sa>sb || (sa==sb && a<b);});
        sos << "Sampled-step feature stats (top " << k << " of " << _n << ", evals=" << _total_evals << ", batches=" << _flushed_batches << "):" << std::endl;
        for(DIMTYPE i=0;i<k;i++) {
                const DIMTYPE f=idx[i];
                sos << " F:" << f << " score=" << score(f)
                    << " m_is=" << _fs[f].m_is << " m_isnot=" << _fs[f].m_isnot
                    << " n_is=" << _fs[f].n_is << " n_isnot=" << _fs[f].n_isnot << std::endl;
        }
        syncout::print(os,sos);
        return true;
}
 
} // namespace
#endif // FSTRESULTWEIGHTERSAMPLED_H ///:~