Collects multiple results to evaluate various stability measures on the collection. More...

#include <result_tracker_stabileval.hpp>

Inheritance diagram for FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >:

Public Types
typedef Result_Tracker< RETURNTYPE, SUBSET >	parent
	the abstract Result_Tracker interface this class implements
typedef Clonable_Impl< Result_Tracker_Stability_Evaluator, parent, CLONE_INHERIT, CLONE_INHERIT, CLONE_DEEP >	parent_impl
	implements stateless_clone() via the copy-constructor (CLONE_DEEP); clone()/sharing_clone() are inherited from Result_Tracker, which throws for both
typedef parent::ResultRec	ResultRec
	see Result_Tracker::ResultRec
typedef ResultRec *	PResultRec
	raw pointer to a ResultRec, as used by get_first()/get_next() iteration
typedef std::shared_ptr< SUBSET >	PSubset
typedef std::list< ResultRec >::const_iterator	CONSTRESULTSITER
	read-only iterator over results, used throughout the stability/similarity measure computations
typedef std::list< ResultRec >::iterator	RESULTSITER
	mutable iterator over results, as returned by begin()/end()
Public Types inherited from FST::Result_Tracker< RETURNTYPE, SUBSET >
typedef std::shared_ptr< SUBSET >	PSubset
	shared pointer to a tracked feature subset
typedef ResultRec *	PResultRec
	raw pointer to a ResultRec, as used by get_first()/get_next() iteration

Public Member Functions
	Result_Tracker_Stability_Evaluator (const IDXTYPE capacity_limit=0)
	Result_Tracker_Stability_Evaluator (const Result_Tracker_Stability_Evaluator &rtse)
	copies the capacity limit only; the `results` list is NOT copied (starts empty) – this constructor backs stateless_clone() (CLONE_DEEP), which by contract does not replicate exact object state
virtual bool	add (const RETURNTYPE value, const PSubset sub) override
	records one (value, subset) result; returns false if the tracker declines to keep it (concrete trackers may restrict by subset size, a value margin from the best known result, or a capacity limit – see the concrete tracker for its actual inclusion rule)
virtual void	clear () override
	discards all recorded results
virtual std::size_t	size () const override
	number of results currently held (std::size_t rather than 'long': on LLP64 platforms 'long' is 32 bits and would narrow the container's own size type)
virtual bool	get_first (PResultRec &r) override
	iterator
virtual bool	get_next (PResultRec &r) override
	iterator
RETURNTYPE	value_mean () const
	mean value of the ResultRec::value values; throws fst_error on an empty collection
RETURNTYPE	value_stddev () const
	standard deviation of the ResultRec::value values; throws fst_error on an empty collection
RETURNTYPE	size_mean () const
	mean value of the size of ResultRec::sub subsets; throws fst_error on an empty collection
RETURNTYPE	size_stddev () const
	standard deviation of the size of ResultRec::sub subsets; throws fst_error on an empty collection
RETURNTYPE	stability_C () const
	stability measure "C" (consistency: mean over features of the relative frequency of co-occurrence in the held subsets); [0,1], higher = more stable
RETURNTYPE	stability_CW () const
RETURNTYPE	stability_CWrel (const DIMTYPE Ysiz) const
	stability measure "CW_{rel}": "CW" rescaled between the minimum and maximum attainable for the observed number of subsets and their total cardinality, which removes the dependence on subset size and on the size of Y. Range [0,1], higher = more stable.
RETURNTYPE	stability_SH () const
RETURNTYPE	stability_PH (const DIMTYPE Ysiz) const
	stability measure "PH" (relative Hamming distance between the held subsets, expressed as agreement). Range [0,1], higher = more stable.
RETURNTYPE	stability_ATI () const
RETURNTYPE	stability_ANHI (const DIMTYPE Ysiz) const
	stability measure "ANHI" (average pairwise normalized Hamming index over all pairs of held subsets). Range [0,1], higher = more stable.
RETURNTYPE	similarity_IC (const Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET > &rtse) const
	similarity measure "IC", the two-collection counterpart of stability_C(); [0,1], higher = more similar
RETURNTYPE	similarity_ICW (const Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET > &rtse) const
	similarity measure "ICW", the two-collection counterpart of stability_CW(); [0,1], higher = more similar
RETURNTYPE	similarity_IATI (const Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET > &rtse) const
RETURNTYPE	similarity_IANHI (const DIMTYPE Ysiz, const Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET > &rtse) const
	similarity measure "IANHI": mean normalized Hamming index over all cross pairs (one subset from each collection). Range [0,1], higher = more similar.
virtual std::ostream &	print (std::ostream &os) const override
	writes a human-readable identification of the object; base default writes nothing
RESULTSITER	begin ()
	iterator to the best (first) held result
RESULTSITER	end ()
	iterator past the last held result
Public Member Functions inherited from FST::Clonable_Impl< Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >, Result_Tracker< RETURNTYPE, SUBSET >, CLONE_INHERIT, CLONE_INHERIT, CLONE_DEEP >
Result_Tracker< RETURNTYPE, SUBSET > *	clone () const override
	dispatches to the DEEP_KIND mechanism (default CLONE_DEEP: Derived's copy-constructor)
Result_Tracker< RETURNTYPE, SUBSET > *	sharing_clone () const override
	dispatches to the SHARING_KIND mechanism (default CLONE_INHERIT: defers to Base, which throws by family convention)
Result_Tracker< RETURNTYPE, SUBSET > *	stateless_clone () const override
	dispatches to the STATELESS_KIND mechanism (default CLONE_INHERIT: defers to Base, which throws by family convention)
Public Member Functions inherited from FST::Result_Tracker< RETURNTYPE, SUBSET >
	Result_Tracker (const Result_Tracker &rt)
	copies the output-detail setting only – concrete trackers copy their own collected results separately
virtual void	join (Result_Tracker &src)
	adds the contents of `src` to this tracker, one add() per record, so this tracker's own inclusion rules apply; `src` is only read, but its iteration cursor is consumed
Result_Tracker *	clone () const override
Result_Tracker *	sharing_clone () const override
void	set_output_detail (OutputDetail detail)
	sets the amount of information that is logged to output stream throughout the course of search
OutputDetail	get_output_detail () const
	the currently configured output detail level
bool	output_normal () const
	true unless output detail is SILENT
bool	output_detailed () const
	true only at the DETAILED output level
Public Member Functions inherited from FST::Clonable
bool	is_clone () const
bool	is_sharing () const
	true if this instance is a sharing_clone() result (referencing, not owning, the source's structures)
bool	is_sharing_clone () const
	true if this instance is BOTH a clone and sharing – i.e. specifically a sharing_clone() result, not a clone()/stateless_clone() one

Protected Member Functions
RETURNTYPE	get_TI (const PSubset &sub1, const PSubset &sub2) const
	pairwise Tanimoto-style index of `sub1` and `sub2:` (size of selected-feature intersection) / (size of selected-feature union), over a Ysiz=max(sub1,sub2 raw size) feature universe; -1.0 (with a warning) if the union is empty. Feeds stability_ATI()/similarity_IATI()
RETURNTYPE	get_NHI (const DIMTYPE Ysiz, const PSubset &sub1, const PSubset &sub2) const
	pairwise normalized Hamming index of `sub1` and `sub2` over an explicit `Ysiz-feature` universe: 1 - (symmetric-difference size)/Ysiz. Feeds stability_ANHI()/similarity_IANHI()
DIMTYPE	max_features () const
	size of the feature universe the held results were selected from: the largest sub->get_n_raw() over the collection (NOT the largest subset size); 0 when the collection is empty
Protected Member Functions inherited from FST::Clonable
void	set_cloned (const bool cloned=true)
	marks/unmarks this instance as a clone; call from clone()/stateless_clone() overrides (Clonable_Impl does this automatically)
void	set_sharing (const bool sharing=true)
	marks/unmarks this instance as sharing (see is_sharing())
void	set_sharing_cloned (const bool cloned=true, const bool sharing=true)
	sets both flags at once; call from sharing_clone() overrides (Clonable_Impl does this automatically)

Protected Attributes
std::list< ResultRec >	results
	held results, sorted descending by value (ties broken toward the smaller subset); duplicates are permitted (see the class note)
const IDXTYPE	_capacity_limit
	set by the constructor; 0 = unlimited
Protected Attributes inherited from FST::Result_Tracker< RETURNTYPE, SUBSET >
OutputDetail	_detail
	the output detail level set via set_output_detail(); read by output_normal()/output_detailed()

Private Attributes
RESULTSITER	getiter
	iteration cursor for get_first()/get_next() (single shared cursor, not reentrant)
std::vector< int >	_ti_mask
	get_TI() scratch: per-feature selection count (0/1/2) over the current pair
std::vector< int >	_nhi_mask
	get_NHI() scratch: per-feature selection count (0/1/2) over the current pair
std::vector< int >	_sim_F1
	similarity_IC()/similarity_ICW() scratch: per-feature selection count over this tracker's results
std::vector< int >	_sim_F2
	similarity_IC()/similarity_ICW() scratch: per-feature selection count over the other tracker's results

Detailed Description

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>
class FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >

Collects multiple results to evaluate various stability measures on the collection.

Feed one feature subset per selection trial (different data splits, different bootstrap samples, repeated randomized runs) with add(), then read a stability measure off the collection, or compare two such collections with the similarity measures. All eleven measures are normalized to the interval [0,1], and in all of them a HIGHER value means more stable (respectively more similar) output; each declaration below states the convention again.

The measures and their properties are defined in P. Somol, J. Novovičová: Evaluating Stability and Comparing Output of Feature Selectors that Optimize Feature Subset Cardinality, IEEE Transactions on Pattern Analysis and Machine Intelligence 32(11), 2010, pp. 1921–1939, except where a declaration names a different source.

Note: Duplicates are permitted.; Setting nonzero capacity limit here should be avoided unless absolutely necessary, otherwise the stability measure value computation would be based on incomplete (and possibly misleading) information; The stability and similarity measures are defined on an empty collection: they return 0.0 (the similarity ones also emit a warning). The descriptive statistics value_mean(), value_stddev(), size_mean() and size_stddev(), however, THROW fst_error when the collection is empty — test size() first.

Examples: demo55.cpp.

Member Typedef Documentation

◆ PSubset

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

typedef std::shared_ptr<SUBSET> FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::PSubset

shared pointer to a tracked feature subset

Constructor & Destructor Documentation

◆ Result_Tracker_Stability_Evaluator()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::Result_Tracker_Stability_Evaluator ( const IDXTYPE capacity_limit = 0 )

inline

Parameters

capacity_limit if nonzero, caps the number of retained results (the tail, i.e. the worst-valued results, is trimmed as new results are inserted); 0 = unlimited – avoid a nonzero limit here unless necessary, as it biases the stability measures towards incomplete information (see the class note)

Member Function Documentation

◆ add()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

bool FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::add	(	const RETURNTYPE	value,
		const PSubset	sub )

overridevirtual

records one (value, subset) result; returns false if the tracker declines to keep it (concrete trackers may restrict by subset size, a value margin from the best known result, or a capacity limit – see the concrete tracker for its actual inclusion rule)

Implements FST::Result_Tracker< RETURNTYPE, SUBSET >.

◆ clear()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

virtual void FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::clear ( )

inlineoverridevirtual

discards all recorded results

Implements FST::Result_Tracker< RETURNTYPE, SUBSET >.

◆ get_first()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

bool FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::get_first ( PResultRec & r )

overridevirtual

iterator

Implements FST::Result_Tracker< RETURNTYPE, SUBSET >.

◆ get_next()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

bool FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::get_next ( PResultRec & r )

overridevirtual

iterator

Implements FST::Result_Tracker< RETURNTYPE, SUBSET >.

◆ print()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

virtual std::ostream & FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::print ( std::ostream & os ) const

inlineoverridevirtual

writes a human-readable identification of the object; base default writes nothing

Reimplemented from FST::Clonable.

◆ similarity_IANHI()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

RETURNTYPE FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::similarity_IANHI	(	const DIMTYPE	Ysiz,
		const Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET > &	rtse ) const

similarity measure "IANHI": mean normalized Hamming index over all cross pairs (one subset from each collection). Range [0,1], higher = more similar.

Parameters

Ysiz	size of the feature universe Y (must be >0)
rtse	the other collection

◆ similarity_IATI()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

RETURNTYPE FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::similarity_IATI ( const Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET > & rtse ) const

similarity measure "IATI": mean Tanimoto index over all cross pairs (one subset from each collection); [0,1], higher = more similar

◆ size()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

virtual std::size_t FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::size ( ) const

inlineoverridevirtual

number of results currently held (std::size_t rather than 'long': on LLP64 platforms 'long' is 32 bits and would narrow the container's own size type)

Implements FST::Result_Tracker< RETURNTYPE, SUBSET >.

◆ stability_ANHI()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

RETURNTYPE FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::stability_ANHI ( const DIMTYPE Ysiz ) const

stability measure "ANHI" (average pairwise normalized Hamming index over all pairs of held subsets). Range [0,1], higher = more stable.

Parameters

Ysiz	size of the feature universe Y (must be >0)

◆ stability_ATI()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

RETURNTYPE FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::stability_ATI ( ) const

stability measure "ATI" (average pairwise Tanimoto index over all pairs of held subsets); [0,1], higher = more stable. Cf. A. Kalousis, J. Prados, and M. Hilario, “Stability of feature selection algorithms,” in Proc. of 5th IEEE International Conference on Data Mining (ICDM 05), Houston, Texas, 2005, pp. 218–225

◆ stability_CW()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

RETURNTYPE FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::stability_CW ( ) const

stability measure "CW" (weighted consistency: as "C", but each feature contributes proportionally to how often it was selected); [0,1], higher = more stable

◆ stability_CWrel()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

RETURNTYPE FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::stability_CWrel ( const DIMTYPE Ysiz ) const

stability measure "CW_{rel}": "CW" rescaled between the minimum and maximum attainable for the observed number of subsets and their total cardinality, which removes the dependence on subset size and on the size of Y. Range [0,1], higher = more stable.

Parameters

Ysiz	size of the feature universe Y (must be >0)

◆ stability_PH()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

RETURNTYPE FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::stability_PH ( const DIMTYPE Ysiz ) const

stability measure "PH" (relative Hamming distance between the held subsets, expressed as agreement). Range [0,1], higher = more stable.

Parameters

Ysiz	size of the feature universe Y (must be >0)

◆ stability_SH()

template<class RETURNTYPE, typename IDXTYPE, typename DIMTYPE, class SUBSET>

RETURNTYPE FST::Result_Tracker_Stability_Evaluator< RETURNTYPE, IDXTYPE, DIMTYPE, SUBSET >::stability_SH ( ) const

stability measure "SH" (Shannon-entropy based); [0,1], higher = more stable. Cf. Somol, Novovičová, Pudil: A New Measure of Feature Selection Algorithms’ Stability, 2009 IEEE International Conference on Data Mining Workshops

The documentation for this class was generated from the following file:

/Users/psms/dev/devtest1/_src_search/result_tracker_stabileval.hpp

Public Types

Public Member Functions

Protected Member Functions

Protected Attributes

Private Attributes

Detailed Description

Member Typedef Documentation

◆ PSubset

Constructor & Destructor Documentation

◆ Result_Tracker_Stability_Evaluator()

Member Function Documentation

◆ add()

◆ clear()

◆ get_first()

◆ get_next()

◆ print()

◆ similarity_IANHI()

◆ similarity_IATI()

◆ size()

◆ stability_ANHI()

◆ stability_ATI()

◆ stability_CW()

◆ stability_CWrel()

◆ stability_PH()

◆ stability_SH()