criterion_extern_value.hpp

Go to the documentation of this file.

#ifndef FSTCRITERIONEXTERNVALUE_H
#define FSTCRITERIONEXTERNVALUE_H
 /*
=========================================================================
Copyright and license:
  * FST4 software (with the exception of any bundled or externally linked
    third-party components) is Copyright (C) 2000-present Institute of
    Information Theory and Automation of the Czech Academy of Sciences (UTIA).
  * FST4 is free software: you can redistribute it and/or modify it under
    the terms of the GNU Affero General Public License as published by the
    Free Software Foundation, version 3. The full text of the license is in
    the file LICENSE in the distribution root and at
    https://www.gnu.org/licenses/agpl-3.0.html.
  * FST4 is distributed in the hope that it will be useful, but WITHOUT ANY
    WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public
    License for more details.
  * Closed-source use without the AGPL's copyleft obligations requires a
    commercial license from UTIA; in case of interest write to
    utia@utia.cas.cz. LICENSING.md in the distribution root explains both
    options, the contributor license agreement that every contribution
    requires, and the contacts (technical questions and pull requests:
    somol@utia.cas.cz, with "FST" in the subject line).
========================================================================== */
 
#include <memory>
#include <vector>
#include <string>
#include <cstdio>
#include <cstdlib>
#include <cassert>
#include <cmath>
#include <iostream>
#include "error.hpp"
#include "global.hpp"
#include "clonable.hpp"
#include "criterion.hpp"
#include "extern_bridge.hpp"
 
/*============== Template parameter type naming conventions ==============
--------- Numeric types: -------------------------------------------------
DATATYPE - data sample values - usually real numbers (but may be integers
          in text processing etc.)
REALTYPE - must be real numbers - for representing intermediate results of
          calculations like mean, covariance etc.
IDXTYPE - index values for enumeration of data samples - (nonnegative) integers,
          extent depends on numbers of samples in data
DIMTYPE - index values for enumeration of features (dimensions), or classes (not
          class sizes) - (nonnegative) integers, usually lower extent than IDXTYPE,
          but be aware of expressions like _classes*_features*_features !
          in linearized representations of feature matrices for all classes
BINTYPE - feature selection marker type - represents ca. <10 different feature
          states (selected, deselected, sel./desel. temporarily 1st nested loop, 2nd...)
RETURNTYPE - criterion value: real value, but may be extended in future to support
          multiple values
--------- Class types: ---------------------------------------------------
SUBSET       - class of class type Subset
CLASSIFIER   - class implementing interface defined in abstract class Classifier
EVALUATOR    - class implementing interface defined in abstract class Sequential_Step
DISTANCE     - class implementing interface defined in abstract class Distance
DATAACCESSOR - class implementing interface defined in abstract class Data_Accessor
INTERVALCONTAINER - class of class type Data_Intervaller
CONTAINER    - STL container of class type Data_Interval
========================================================================== */
 
namespace FST {

template<class RETURNTYPE, typename DATATYPE, typename REALTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET, class DATAACCESSOR>
class Criterion_Extern_Value : public Clonable_Impl<Criterion_Extern_Value<RETURNTYPE,DATATYPE,REALTYPE,IDXTYPE,DIMTYPE,SUBSET,DATAACCESSOR>,Criterion<RETURNTYPE,SUBSET> > {
public:
        typedef Criterion<RETURNTYPE,SUBSET> parent; 
        typedef Clonable_Impl<Criterion_Extern_Value,parent> parent_impl; 
        typedef std::shared_ptr<DATAACCESSOR> PDataAccessor;  
        typedef std::shared_ptr<SUBSET> PSubset;              
        typedef typename DATAACCESSOR::PPattern PPattern;     
        Criterion_Extern_Value() : _bridge("Criterion_Extern_Value"), _monotonic(false), _no_data(false), _data_stale(true), _classes(0), _features(0), _subfeatures(0) {notify("Criterion_Extern_Value constructor.");}
        virtual ~Criterion_Extern_Value() {notify("Criterion_Extern_Value destructor.");} // the bridge destructor shuts the child down
 
        void set_command(const std::string &cmd) {_bridge.set_command(cmd);} 
        const std::string& get_command() const {return _bridge.get_command();} 
        void set_monotonic(const bool m) {_monotonic=m;} 
        virtual bool is_monotonic() const override {return _monotonic;}
        void set_no_data(const bool nd) {_no_data=nd;} 

        void initialize(const PDataAccessor da, const PDataAccessor da_for_evaluate=PDataAccessor());
        virtual bool evaluate(RETURNTYPE &result, const PSubset sub) override;
 
        virtual std::ostream& print(std::ostream& os) const override {os << "Criterion_Extern_Value(cmd=\"" << _bridge.get_command() << "\", monotonic=" << (_monotonic?"asserted":"no") << (_no_data?", no-data":"") << ")"; return os;}
private:
        friend parent_impl; 
        Criterion_Extern_Value(const Criterion_Extern_Value& cev) :
                _bridge("Criterion_Extern_Value"), // clones NEVER share pipes - each fresh bridge spawns its own child lazily
                _monotonic(cev._monotonic),
                _no_data(cev._no_data),
                _data_stale(true), // the clone's own child holds no DATA yet
                _da(cev._da),
                _classes(cev._classes),
                _features(cev._features),
                _subfeatures(cev._subfeatures),
                _index(cev._index)
        {
                _bridge.set_command(cev._bridge.get_command());
                notify("Criterion_Extern_Value copy-constructor.");
        }
protected:
        Extern_Bridge _bridge;         
        bool _monotonic;               
        bool _no_data;                 
        bool _data_stale;              
        PDataAccessor _da;             
        DIMTYPE _classes;              
        DIMTYPE _features;             
        DIMTYPE _subfeatures;          
        std::vector<DIMTYPE> _index;   
 
        bool send_data_block();        
        void send_row(const DIMTYPE label, const PPattern &pattern); 
        double parse_value(const std::string &line); 
};
 
template<class RETURNTYPE, typename DATATYPE, typename REALTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET, class DATAACCESSOR>
void Criterion_Extern_Value<RETURNTYPE,DATATYPE,REALTYPE,IDXTYPE,DIMTYPE,SUBSET,DATAACCESSOR>::initialize(const PDataAccessor da, const PDataAccessor)
{
        notify("Criterion_Extern_Value::initialize().");
        assert(da);
        assert(da->getNoOfFeatures()>0);
        // the class-count requirement guards the DATA protocol (its header carries K, its rows carry
        // labels); in no-data mode the tool never sees either, and the data-free `--features` entry
        // legitimately supplies a classless universe accessor — so the check applies only when DATA flows
        if(!_no_data && da->getNoOfClasses()<2) throw fst_error("Criterion_Extern_Value requires at least 2 classes");
        if(_bridge.get_command().empty()) throw fst_error("Criterion_Extern_Value: no command set (see --extern-cmd)");
        _da=da;
        _classes=da->getNoOfClasses();
        _features=da->getNoOfFeatures();
        _index.resize(_features);
        _subfeatures=0;
        _data_stale=true; // the pipeline calls initialize() whenever the effective split changes
                          // (the filter re-estimation contract) - the tool's DATA must follow
}
 
template<class RETURNTYPE, typename DATATYPE, typename REALTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET, class DATAACCESSOR>
void Criterion_Extern_Value<RETURNTYPE,DATATYPE,REALTYPE,IDXTYPE,DIMTYPE,SUBSET,DATAACCESSOR>::send_row(const DIMTYPE label, const PPattern &pattern)
{
        std::FILE *out=_bridge.to_child();
        char buf[64];
        std::snprintf(buf,sizeof(buf),"%ld",(long)label); std::fputs(buf,out);
        for(DIMTYPE f=0;f<_features;f++) // FULL width: the DATA block carries every feature
        {
                std::snprintf(buf,sizeof(buf),"%.17g",(double)pattern[f]); // bit round-trip for doubles
                std::fputc(' ',out);
                std::fputs(buf,out);
        }
        if(std::fputc('\n',out)==EOF)
                _bridge.die_on_write();
}
 
template<class RETURNTYPE, typename DATATYPE, typename REALTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET, class DATAACCESSOR>
bool Criterion_Extern_Value<RETURNTYPE,DATATYPE,REALTYPE,IDXTYPE,DIMTYPE,SUBSET,DATAACCESSOR>::send_data_block()
{
        // header first: count the training patterns (cheap block-size sweep)
        unsigned long long n=0;
        {IDXTYPE s; for(DIMTYPE c=0;c<_classes;c++) {_da->setClass(c); for(const auto &blk : _da->blocks(TRAIN)) {s=blk.size; n+=(unsigned long long)s;}}}
        if(n==0) return false;
        _bridge.send_line("DATA "+std::to_string(n)+" "+std::to_string((long)_features)+" "+std::to_string((long)_classes));
 
        typename DATAACCESSOR::PPattern p;
        IDXTYPE s,i;
        const DIMTYPE stride=_da->getNoOfFeatures();
        for(DIMTYPE c=0;c<_classes;c++)
        {
                _da->setClass(c);
                for(const auto &blk : _da->blocks(TRAIN)) for(i=0,p=blk.data,s=blk.size;i<s;i++)
                        send_row(c,&p[(std::size_t)i*stride]);
        }
        _bridge.flush_to_child();
        const std::string resp=_bridge.read_line();
        if(resp!="OK") throw fst_error("Criterion_Extern_Value: '"+_bridge.get_command()+"' rejected DATA: "+resp);
        _data_stale=false;
        return true;
}
 
template<class RETURNTYPE, typename DATATYPE, typename REALTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET, class DATAACCESSOR>
double Criterion_Extern_Value<RETURNTYPE,DATATYPE,REALTYPE,IDXTYPE,DIMTYPE,SUBSET,DATAACCESSOR>::parse_value(const std::string &line)
{
        // strict full-line numeric parse: bare strtod would accept any garbage line (stray
        // child diagnostics, "ERROR ..." messages) as a valid 0.0 value - silently wrong
        // search results; fail loudly instead, like the classifier bridge's prediction parse
        const char *s=line.c_str();
        char *end=0;
        const double v=std::strtod(s,&end);
        if(end==s || *end!='\0')
                {_bridge.shutdown_process(); throw fst_error("Criterion_Extern_Value: '"+_bridge.get_command()+"' returned a non-numeric value line: '"+line+"'");}
        // strtod accepts "nan"/"inf" as numbers, but a non-finite criterion value would poison
        // every comparison SILENTLY (NaN orders false against everything - subsets never win,
        // B&B cut logic misbehaves invisibly); reject it as loudly as garbage text
        if(!std::isfinite(v))
                {_bridge.shutdown_process(); throw fst_error("Criterion_Extern_Value: '"+_bridge.get_command()+"' returned a non-finite value: '"+line+"' (criterion values must be finite; negate error-style measures inside the tool rather than returning infinities)");}
        return v;
}
 
template<class RETURNTYPE, typename DATATYPE, typename REALTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET, class DATAACCESSOR>
bool Criterion_Extern_Value<RETURNTYPE,DATATYPE,REALTYPE,IDXTYPE,DIMTYPE,SUBSET,DATAACCESSOR>::evaluate(RETURNTYPE &result, const PSubset sub)
{
        notify("Criterion_Extern_Value::evaluate().");
        assert(_da);
        assert(sub);
        assert(_classes==_da->getNoOfClasses());
        assert(_features==_da->getNoOfFeatures());
        assert(sub->get_n_raw()==_da->getNoOfFeatures());
 
        // buffer the feature subset index
        _subfeatures=0;
        for(DIMTYPE fsel : sub->selected_features()) {assert(_subfeatures<_features); _index[_subfeatures++]=fsel;}
        assert(_subfeatures==sub->get_d_raw());
        if(_subfeatures==0) return false;
 
        const bool was_running=_bridge.running();
        _bridge.ensure_process();
        if(!was_running) _data_stale=true; // a freshly (re)spawned child holds no DATA
 
        if(!_no_data && _data_stale && !send_data_block()) return false; // empty train part
        // (in no-data mode the tool never receives DATA - it computes from the indices alone)
 
        // one VALUE request: d + the 0-based raw feature indices (ascending subset order)
        std::string req="VALUE "+std::to_string((long)_subfeatures);
        for(DIMTYPE f=0;f<_subfeatures;f++) {req+=' '; req+=std::to_string((long)_index[f]);}
        _bridge.send_line(req);
        _bridge.flush_to_child();
        result=(RETURNTYPE)parse_value(_bridge.read_line());
        return true;
}
 
} // namespace
#endif // FSTCRITERIONEXTERNVALUE_H ///:~