reporter.h

/**************************************************************************
 * Copyright (c) 2001-2011 T. M. Murali                                   *
 * Copyright (c) 2011 Phillip Whisenhunt                                  *
 * Copyright (c) 2011 David Badger                                        *
 * Copyright (c) 2010 Jacqueline Addesa                                   *
 *                                                                        *
 * This file is part of Biorithm.                                         *
 *                                                                        *
 * Biorithm is free software: you can redistribute it and/or modify       *
 * it under the terms of the GNU General Public License as published by   *
 * the Free Software Foundation, either version 3 of the License, or      *
 * (at your option) any later version.                                    *
 *                                                                        *
 * Biorithm is distributed in the hope that it will be useful,            *
 * but WITHOUT ANY WARRANTY; without even the implied warranty of         *
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the          *
 * GNU General Public License for more details.                           *
 *                                                                        *
 * You should have received a copy of the GNU General Public License      *
 * along with Biorithm.  If not, see <http://www.gnu.org/licenses/>.      *
 *                                                                        *
 **************************************************************************/

#ifndef _REPORTER_H
#define _REPORTER_H

#include <map>
#include <string>
#include <fstream>
#include <iostream>
#include <sys/stat.h>


using namespace std;

#include "boost/multi_index_container.hpp"
#include "boost/multi_index/member.hpp"
#include "boost/multi_index/ordered_index.hpp"
#include <boost/multi_index/composite_key.hpp>

using boost::multi_index_container;
using namespace boost::multi_index;
  
#include "biofunction.h"
#include "graph-global.h" //for MyNT
#include "old-annotations.h"

#include "histogram.h"

// i want to get rid of the CVResult typedef and everything that depends on it.
#undef USE_CVRESULT

enum MyGainCVResult { TRUE_POSITIVE = 1, FALSE_POSITIVE, TRUE_NEGATIVE, FALSE_NEGATIVE };

struct DetailedCVResult
{
//private:
public:
  string function;
  string gene;
  MyGainAnnotationType correctState;
  MyGainAnnotationType predictedState;
  // confidence in the correct state. this is used by sinksource with evidence code weights.
  MyNT correctStateConfidence;
  MyNT predictionConfidence;
  string algorithm;
public:
  DetailedCVResult()
      : function(), gene(), correctState(), predictedState(), correctStateConfidence(1), predictionConfidence(0), algorithm()
    {}
  // Constructor that takes values for all fields.
  DetailedCVResult(string g, string f, MyGainAnnotationType cs, 
                   MyGainAnnotationType ps, MyNT csConf, MyNT conf, string algo)
      : function(f), gene(g), correctState(cs), predictedState(ps), correctStateConfidence(csConf), predictionConfidence(conf), algorithm(algo)
    {}
  
  string getGene() const
    {
      return(gene);
    }
  
  MyGainAnnotationType getCorrectState() const
    {
      return(correctState);
    }
  
  MyGainAnnotationType getPredictedState() const
    {
      return(predictedState);
    }
  MyNT getPredictionConfidence() const
    {
      return(predictionConfidence);
    }
  
  
  MyGainCVResult computeResult() const
    {
      if (ANNOTATED_STATE == correctState)
        {
          // +1 -> 0 -> +1 is a TP.
          if (ANNOTATED_STATE == predictedState)
            return(TRUE_POSITIVE);
          // +1 -> 0 -> 0 (-1) is a FN.
          if (HYPOTHETICAL_STATE == predictedState)
            return(FALSE_NEGATIVE);
          // +1 -> 0 -> -1 is a FN.
          if (NOT_ANNOTATED_STATE == predictedState)
            return(FALSE_NEGATIVE);
        }
      else if (NOT_ANNOTATED_STATE == correctState)
        {
          // -1 -> 0 -> 1 is a FP.
          if (ANNOTATED_STATE == predictedState)
            return(FALSE_POSITIVE);
          // -1 -> 0 -> 0 (-1) is a TN.
          if (HYPOTHETICAL_STATE == predictedState)
            return(TRUE_NEGATIVE);
          // -1 -> 0 -> -1 is a TN.
          if (NOT_ANNOTATED_STATE == predictedState)
            return(TRUE_NEGATIVE);
        }
      cerr << "DetailedCVResult::computeResult() ERROR: correctState appears to be HYPOTHETICAL_STATE, which is a big no-no" << endl;
    }
  


};


// tell doxygen to skip
struct detailedcvresult_gene_function_correctstate_predictedstate_confidence_algorithm_index {};
struct detailedcvresult_algorithm_confidence_index {};
typedef multi_index_container
<
  DetailedCVResult,
  indexed_by
  <
    // the primary key is made up of gene, function, correct state,
    // predicted state, confidence, and algorithm.
    ordered_unique
    <
      tag< detailedcvresult_gene_function_correctstate_predictedstate_confidence_algorithm_index >,
      composite_key
      <
        DetailedCVResult,
        member< DetailedCVResult, std::string, &DetailedCVResult::gene >,
        member< DetailedCVResult, std::string, &DetailedCVResult::function >,
        member< DetailedCVResult, MyGainAnnotationType, &DetailedCVResult::correctState >,
        member< DetailedCVResult, MyGainAnnotationType, &DetailedCVResult::predictedState >,
        member< DetailedCVResult, MyNT, &DetailedCVResult::predictionConfidence >,
        member< DetailedCVResult, std::string, &DetailedCVResult::algorithm >
        >,
      // specify how to order each field of the primary key.
      composite_key_compare
      <
        std::less< std::string >,
        std::less< std::string >,
        std::less< MyGainAnnotationType >,
        std::less< MyGainAnnotationType >,
        std::greater< MyNT >,
        std::less< std::string >
        >
      >,
    // index by algorithm and confidence.
    ordered_non_unique
    <
      tag< detailedcvresult_algorithm_confidence_index >,
      composite_key
      <
        DetailedCVResult,
        member< DetailedCVResult, std::string, &DetailedCVResult::algorithm >,
        member< DetailedCVResult, MyNT, &DetailedCVResult::predictionConfidence >
        >,
      composite_key_compare
      <
        std::less< std::string >,
        // sort in decreasing order of prediction confidence.
        std::greater< MyNT >
        >
      >
    >
  > SetOfDetailedCVResults;




#ifdef USE_CVRESULT
struct CVResult {
//private:
public:
  int TP, TN, FP, FN;
  MyNT fscore, precision, recall;
  // correct initialisation is 0, otherwise operator+ will not work.

public:
  CVResult() { TP=TN=FP=FN=0;fscore=precision=recall=0.0; };
  CVResult(int tp, int tn, int fp, int fn)
    {
      TP=tp;TN=tn;FP=fp;FN=fn;
      calculate();
    };
  CVResult(const CVResult &other)
    {
      TP= other.TP;
      TN= other.TN;
      FP= other.FP;
      FN= other.FN;
      fscore = other.fscore;
      precision = other.precision;
      recall = other.recall;
    };

  MyNT getFscore() const
    {
      return(fscore);
    }

  MyNT getPrecision() const
    {
      return(precision);
    }

  MyNT getRecall() const
    {
      return(recall);
    }
  
  void calculate()
    {
      precision = (0!=TP+FP) ? (((1.0)*TP)/((1.0)*(TP+FP))) : 0.0;
      recall = (0!=TP+FN) ? (((1.0)*TP)/((1.0)*(TP+FN))) : 0.0;
      fscore = (0 !=precision+recall) ? ((2.0*precision*recall)/(precision+recall)) : 0.0;
    }
  // add two results. just do a field-wise addition.
  CVResult operator+(CVResult &rhs)
    {
      CVResult result(*this);
      result.TP += rhs.TP;
      result.TN += rhs.TN;
      result.FP += rhs.FP;
      result.FN += rhs.FN;      
      result.calculate();
      return(result);
    }
  
  // add rhs to *this. just do a field-wise addition.
  void operator+=(CVResult &rhs)
    {
      TP += rhs.TP;
      TN += rhs.TN;
      FP += rhs.FP;
      FN += rhs.FN;      
      calculate();
    }  
};
#endif // USE_CVRESULT

struct PredictionDetails {
  MyNT energy, prob, input, threshold;
  PredictionDetails(MyNT p, MyNT i, MyNT t, MyNT e = 0)
    { prob=p;input=i,threshold=t; energy = e; };
        PredictionDetails()
    { prob=input=threshold=-2.0; energy = 0;};
};


struct Prediction
{
//  friend typename SetOfPredictions;
public:
  // these need to be public for SetOfPredictions.
  string _gene;
  string _function;
  string _algorithm;
  MyNT _confidence;
  MyNT _cutBasedConfidence;
  // the rank of this prediction in the list of all predictions.
  unsigned int _rankOverall;
  // the rank of this prediction in the list of predictions for this function.
  unsigned int _rankFunction;
  
public:
  Prediction(string g, string f, string a, MyNT c = 0)
      : _gene(g), _function(f), _algorithm(a), _confidence(c), _cutBasedConfidence(0),
        _rankOverall(0), _rankFunction(0)
    {}

  void setCutBasedConfidence(MyNT c)
    {
      _cutBasedConfidence = c;
    }
  MyNT getCutBasedConfidence() const
    {
      return(_cutBasedConfidence);
    }
  
};

// tell doxygen to skip
struct prediction_gene_function_algorithm_index {};
struct prediction_gene_algorithm_index {};
struct prediction_gene_index {};
struct prediction_function_index {};
struct prediction_algorithm_index {};

struct prediction_algorithm_confidence_index {};

struct prediction_algorithm_function_confidence_index {};
typedef multi_index_container<
  Prediction,
  indexed_by<

    // define multi-column primary key.
    ordered_unique<
      tag< prediction_gene_function_algorithm_index >,
      composite_key<
      Prediction,
      member< Prediction, std::string, &Prediction::_gene>,
      member< Prediction, std::string, &Prediction::_function >,
      member< Prediction, std::string, &Prediction::_algorithm >
      >,
      composite_key_compare<
        std::less<std::string>, // genes sorted by default
        std::less<std::string>, // functions sorted by default
        std::less<std::string>   // algorithms sorted by default
        >
      >,
    // there can be many functions for an algorithm-gene pair.
    ordered_non_unique<
      tag< prediction_gene_algorithm_index >,
      composite_key<
      Prediction,
      member< Prediction, std::string, &Prediction::_algorithm >,
      member< Prediction, std::string, &Prediction::_gene>
    >,
      composite_key_compare<
      std::less<std::string>,   // algorithms sorted by default
      std::less<std::string> // genes sorted by default
    >
    >,
    
    // sort on algorithm, function, and confidence
    ordered_non_unique<
      tag< prediction_algorithm_function_confidence_index >,
      composite_key<
      Prediction,
      member< Prediction, std::string, &Prediction::_algorithm >,
      member< Prediction, std::string, &Prediction::_function >,
      member< Prediction, MyNT, &Prediction::_confidence >
      >,
      composite_key_compare<
        std::less<std::string>, // algorithms sorted by default
        std::less<std::string>, // functions sorted by default
        std::greater<MyNT>   // confidence sorted by decreasing
        >
      >,

  // sort by less<string> on _gene
  ordered_non_unique< tag< prediction_gene_index >, member< Prediction ,std::string, &Prediction::_gene > >,

  // sort on _function
  ordered_non_unique< tag< prediction_function_index >, member< Prediction ,std::string, &Prediction::_function > >,
    
  // sort on _algorithm
    ordered_non_unique< tag< prediction_algorithm_index >, member< Prediction ,std::string, &Prediction::_algorithm > >,
  
  // sort on algorithm followed by confidence
  ordered_non_unique<
    tag< prediction_algorithm_confidence_index >,
    composite_key<
      Prediction,
      member< Prediction, std::string, &Prediction::_algorithm >,
      member< Prediction, MyNT, &Prediction::_confidence>
    >,
      composite_key_compare<
      std::less<std::string>,   // algorithms sorted by default
      std::greater< MyNT > // sort confidence in decreasing order.
    >
    >// ,

//   // sort on algorithm followed by overall rank
//   ordered_non_unique<
//     tag< prediction_algorithm_rank_overall_index >,
//     composite_key<
//       Prediction,
//       member< Prediction, std::string, &Prediction::_algorithm >,
//       member< Prediction, unsigned int, &Prediction::_rankOverall>
//     >
//     >,
    
//   // sort on algorithm followed by function followed by within-function rank
//   ordered_non_unique<
//     tag< prediction_algorithm_rank_function_index >,
//     composite_key<
//       Prediction,
//       member< Prediction, std::string, &Prediction::_algorithm >,
//       member< Prediction, std::string, &Prediction::_function >,
//       member< Prediction, unsigned int, &Prediction::_rankFunction>
//     >
//     >

    >
  
  > SetOfPredictions;


#ifdef USE_CVRESULT
// the first level of these maps store the name of the algorithm. the
// second level the name of the function. the third level the
// threshold.
typedef map< MyNT, CVResult > MyCVResultsPerFunction;
typedef map< string, MyCVResultsPerFunction > MyCVResultsPerAlgo;
typedef map< string, MyCVResultsPerAlgo > MyCVResults; 
#endif //  USE_CVRESULT

// each string is a gene.
typedef map< string, DetailedCVResult > MyDetailedCVResultsPerThreshold;
typedef map< MyNT, MyDetailedCVResultsPerThreshold > MyDetailedCVResultsPerFunction;
//typedef map< string, DetailedCVResult > MyDetailedCVResultsPerFunction;
typedef map< string, MyDetailedCVResultsPerFunction > MyDetailedCVResultsPerAlgo;
typedef map< string, MyDetailedCVResultsPerAlgo > MyDetailedCVResults;
//typedef map< string, map< string, map< MyNT, vector< DetailedCVResult > > > > MyDetailedCVResults;

// PerFunction because each such map stores the results for a function.
typedef map< string, PredictionDetails> MyPredictionResultsPerFunction;
typedef map< string, MyPredictionResultsPerFunction > MyPredictionResultsPerAlgo;
typedef map< string, MyPredictionResultsPerAlgo > MyPredictionResults;
typedef map< string, map< string, map< MyNT, int> > > MyPredictionCountsPerFunctionAndThreshold;

enum MyGainCompareFunctionType { COMPARE_FUNCTION_INVALID = -1,
                                 COMPARE_FUNCTION_IDENTICAL = 0,
                                 COMPARE_FUNCTION_DESCENDANT,
                                 COMPARE_FUNCTION_ANCESTOR,
                                 COMPARE_FUNCTION_RELATIVE };

struct MyGainDAGDistanceType
{
private:
  int _up;
  int _down;
  MyGainCompareFunctionType _whichType;

public:
  MyGainDAGDistanceType(int u = -1, int d = -1)
      : _up(u), _down(d), _whichType(computeDistanceType())
    {}
  virtual ~MyGainDAGDistanceType()
    {}

  MyGainCompareFunctionType computeDistanceType() const;
  
  unsigned int getDistance() const
    {
      return(max(0, _up) + max(0, _down));
    }

  MyGainCompareFunctionType getDistanceType() const
    {
      return(_whichType);
    }

  bool operator<(const MyGainDAGDistanceType &other) const;


  void print(ostream &ostr) const
    {
      ostr << _up << "\t" << _down;
    }
};
  
inline ostream &operator<<(ostream &ostr, const MyGainDAGDistanceType &distance)
{
  distance.print(ostr);
  return(ostr);
}


struct PredictionEvaluation //: public Prediction // commented out because of issues with multi_index in versions before Boost 1.34
{
public:
  string _gene;
  string _function;
  string _algorithm;
  MyNT _confidence;

  // the rank of this prediction in the list of all predictions.
  unsigned int _rankOverall;
  // the rank of this prediction in the list of predictions for this function.
  unsigned int _rankFunction;

  string _closestFunction;
  string _closestExperimentallyAnnotatedFunction;
  // distance to closest function. if both _distanceUp and
  // _distanceDown are 0, the closest function is the predicted
  // function. if only _distanceUp is <= 0, _closestFunction is a
  // descendant of the predicted function, so the prediction is
  // verified. if only _distanceDown is <= 0, _closestFunction is an
  // ancestor of the predictd function. if both are positive, the
  // predicted function is a relative of _closestFunction. if both are
  // negative, something is wrong.
  MyGainDAGDistanceType _distance;
  MyGainDAGDistanceType _distanceToExperimentallyAnnotatedFunction;
  
  PredictionEvaluation(string g = "", string f = "", string a = "", MyNT c = 0)
  //      : Prediction(g, f, a, c), 
      : _gene(g), _function(f), _algorithm(a), _confidence(c),
        _rankOverall(0), _rankFunction(0),
        _closestFunction(), _closestExperimentallyAnnotatedFunction(), _distance(),
        _distanceToExperimentallyAnnotatedFunction()
    {}
  
  bool isClosestFunctionInitialised() const
    {
      return(COMPARE_FUNCTION_INVALID != _distance.getDistanceType());
    }

  bool isClosestExperimentallyAnnotatedFunctionInitialised() const
    {
      return(COMPARE_FUNCTION_INVALID !=
             _distanceToExperimentallyAnnotatedFunction.getDistanceType());
    }

  bool isVerified() const
    {
      return(isClosestFunctionInitialised()
             && ((COMPARE_FUNCTION_DESCENDANT == _distance.getDistanceType())
                 || (COMPARE_FUNCTION_IDENTICAL == _distance.getDistanceType())));
    }
  bool isExperimentallyVerified() const
    {
      return(isClosestExperimentallyAnnotatedFunctionInitialised()
             && ((COMPARE_FUNCTION_DESCENDANT ==
                  _distanceToExperimentallyAnnotatedFunction.getDistanceType())
                 || (COMPARE_FUNCTION_IDENTICAL ==
                     _distanceToExperimentallyAnnotatedFunction.getDistanceType())));
    }
  

  void updateClosestFunction(string f, int up, int down, bool experimentalEC = false);
  

};

// tell doxygen to skip
struct evaluation_gene_function_algorithm_index {};
struct evaluation_gene_algorithm_index {};
struct evaluation_gene_index {};
struct evaluation_function_index {};
struct evaluation_algorithm_index {};

struct evaluation_algorithm_confidence_index {};
struct evaluation_algorithm_rank_overall_index {};
struct evaluation_algorithm_rank_function_index {};
typedef multi_index_container<
  PredictionEvaluation,
  indexed_by<
    // define multi-column primary key.
    ordered_unique<
      tag< evaluation_gene_function_algorithm_index >,
      composite_key<
      PredictionEvaluation,
      member< PredictionEvaluation, string, &PredictionEvaluation::_gene>,
      member< PredictionEvaluation, string, &PredictionEvaluation::_function >,
      member< PredictionEvaluation, string, &PredictionEvaluation::_algorithm >
      >,
      composite_key_compare<
      std::less<string>, // genes sorted by default
      std::less<string>, // functions sorted by default
      std::less<string>   // algorithms sorted by default
      >
    >,
    // there can be many functions for an algorithm-gene pair.
    ordered_non_unique<
      tag< evaluation_gene_algorithm_index >,
    composite_key<
      PredictionEvaluation,
        member< PredictionEvaluation, string, &PredictionEvaluation::_algorithm >,
        member< PredictionEvaluation, string, &PredictionEvaluation::_gene>
    >,
    composite_key_compare<
        std::less<string>,   // algorithms sorted by default
        std::less<string> // genes sorted by default
  >
>,
    

    
  // sort by less<string> on _gene
  ordered_non_unique< tag< evaluation_gene_index >,
                      member< PredictionEvaluation, string, &PredictionEvaluation::_gene > >,

  // sort on _function
  ordered_non_unique< tag< evaluation_function_index >,
                      member< PredictionEvaluation, string, &PredictionEvaluation::_function > >,
    
  // sort on _algorithm
  ordered_non_unique< tag< evaluation_algorithm_index >,
                      member< PredictionEvaluation, string, &PredictionEvaluation::_algorithm > >,

  // sort on algorithm followed by confidence
  ordered_non_unique<
    tag< evaluation_algorithm_confidence_index >,
    composite_key<
      PredictionEvaluation,
      member< PredictionEvaluation, std::string, &PredictionEvaluation::_algorithm >,
      member< PredictionEvaluation, MyNT, &PredictionEvaluation::_confidence>
    >,
      composite_key_compare<
      std::less<std::string>,   // algorithms sorted by default
      std::greater< MyNT > // sort confidence in decreasing order.
    >
    >,

  // sort on algorithm followed by overall rank
  ordered_non_unique<
    tag< evaluation_algorithm_rank_overall_index >,
    composite_key<
      PredictionEvaluation,
      member< PredictionEvaluation, std::string, &PredictionEvaluation::_algorithm >,
      member< PredictionEvaluation, unsigned int, &PredictionEvaluation::_rankOverall>
    >
    >,
    
  // sort on algorithm followed by function followed by within-function rank
  ordered_non_unique<
    tag< evaluation_algorithm_rank_function_index >,
    composite_key<
      PredictionEvaluation,
      member< PredictionEvaluation, std::string, &PredictionEvaluation::_algorithm >,
      member< PredictionEvaluation, std::string, &PredictionEvaluation::_function >,
      member< PredictionEvaluation, unsigned int, &PredictionEvaluation::_rankFunction>
    >
    >


    
    > 
  > SetOfEvaluations;


struct PredictionEvaluationSummary
{
public:
  string _algorithm;
  // the total number of predictions.
  unsigned int _numTotalPredictions;

  // statistics on what can be verified.
  
  // _numPredictionsForVerifiableGenes counts the number of predictions
  // for verifiable genes. some of the predictions may themselves not
  // be verifiable since the gene does not have a new annotation in
  // the same category as the predicted function. 
  // _numVerifiablePredictions counts the #of verifiable predictions.
  unsigned int _numPredictionsForVerifiableGenes;
  unsigned int _numVerifiablePredictions;
  // numTotalMSPs is just the total number of MSPs. 
  // numMSPsForVerifiableGenes and numVerifiableMSPs are the analogues
  // for MSPs for numPredictionsForVerifiableGenes and
  // numVerifiablePredictions.
  unsigned int _numTotalMSPs, _numMSPsForVerifiableGenes, _numVerifiableMSPs;

  // the next few quantities give an idea of the hierarchical
  // consistency of the predictions. they count the #predictions/#MSPs
  // for various categories of predictions.
  //
  // _numAverageMSPs = _numTotalPredictions/_numTotalMSPs.
  //
  // _numAverageMSPsForVerifiableGenes = _numPredictionsForVerifiableGenes/_numMSPsForVerifiableGenes.
  //
  // _numAverageVerifiableMSPs = _numVerifiablePredictions/_numVerifiableMSPs

  MyNT _numAverageMSPs, _numAverageMSPsForVerifiableGenes, _numAverageVerifiableMSPs;

  unsigned int _numVerifiableGenes;


  // statistics on what is verified.
  
  
  // key is the distance to the verifying new annotating function. 
  // value is the #MSPs with this distance.
  map< unsigned int, unsigned int > _distanceToNumVerifiedMSPs;
  map< unsigned int, unsigned int > _distanceToNumExperimentallyVerifiedMSPs;
  // total of the values in the _distanceToNumVerifiedMSPs map.
  unsigned int _numVerifiedMSPs, _numExperimentallyVerifiedMSPs;
  // the next map stores an entry at "index" d for each gene such that
  // the closest new annotation over all verified predicted functions
  // for that gene is d.
  map< unsigned int, unsigned int > _distanceToNumVerifiedGenes;
  map< unsigned int, unsigned int > _distanceToNumExperimentallyVerifiedGenes;
  set< string > _verifiedGenes, _experimentallyVerifiedGenes;
  // the number of genes in _verifiedGenes, also the total of the
  // values in the _distanceToNumVerifiedGenes map.
  unsigned int _numVerifiedGenes, _numExperimentallyVerifiedGenes;

  // histograms of confidence values.
  
  // the histogram of confidence values for verified MSPs.
  MyHistogram _verifiedMSPsConfidenceHistogram, _experimentallyVerifiedMSPsConfidenceHistogram;
  // the histogram of confidence values for unverified MSPs for genes
  // with at least one verified MSP.
  MyHistogram _verifiedGeneUnverifiedMSPsConfidenceHistogram,
    _experimentallyVerifiedGeneUnverifiedMSPsConfidenceHistogram;
  // the histogram of confidence values for MSPs for genes
  // without any verified MSP.
  MyHistogram _unverifiedGeneMSPsConfidenceHistogram;
  // the histogram of confidence values for MSPs for genes
  // without any experimentally verified MSP.
  MyHistogram _experimentallyUnverifiedGeneMSPsConfidenceHistogram;
  

  // recalls/precisions for ROC curves over all functions.
  MyHistogram _allConfidenceHistogram;
  MyHistogram _correctConfidenceHistogram;
  MyHistogram _precisionHistogram;
  MyHistogram _recallHistogram;
  MyHistogram _oneMinusRecallHistogram;
  
  // per function curves.
  map< string, MyHistogram > _perFunctionAllConfidenceHistograms;
  map< string, MyHistogram > _perFunctionCorrectConfidenceHistograms;
  map< string, unsigned int >_perFunctionNumNewAnnotations;
  map< string, MyHistogram > _perFunctionPrecisionHistograms;
  map< string, MyHistogram > _perFunctionRecallHistograms;
//  map< BioFunction, MyHistogram > _perFunctionPrecisionHistogram;

  MyHistogram _verifiedMSPsRankHistogram, _experimentallyVerifiedMSPsRankHistogram;
  MyHistogram _verifiedGeneUnverifiedMSPsRankHistogram,
    _experimentallyVerifiedGeneUnverifiedMSPsRankHistogram;
  MyHistogram _unverifiedGeneMSPsRankHistogram;
  MyHistogram _experimentallyUnverifiedGeneMSPsRankHistogram;
  MyHistogram _allRankHistogram;
  MyHistogram _correctRankHistogram;
  map< string, MyHistogram > _perFunctionAllRankHistograms;
  map< string, MyHistogram > _perFunctionCorrectRankHistograms;

public:
  
};
  

// struct evaluation_gene_function_algorithm_index {};
// struct evaluation_gene_algorithm_index {};
// struct evaluation_gene_index {};
// struct evaluation_function_index {};
struct evaluation_summaries_algorithm_index {};

typedef multi_index_container<
  PredictionEvaluationSummary,
  indexed_by<
  // sort on _algorithm
  ordered_unique< tag< evaluation_summaries_algorithm_index >,
                      member< PredictionEvaluationSummary, string, &PredictionEvaluationSummary::_algorithm > >
    > 
  > SetOfEvaluationSummaries;

class Reporter
{
private:

  SetOfPredictions _allPredictions;
  SetOfEvaluations _allEvaluations;
  SetOfEvaluationSummaries _allEvaluationSummaries;
  
  SetOfDetailedCVResults _allDetailedCVResults;
  
  MyPredictionResults predictionResults;
#ifdef USE_CVRESULT
  MyCVResults cvResults;
#endif // USE_CVRESULT
  MyDetailedCVResults detailedCvResults;
  MyPredictionCountsPerFunctionAndThreshold predictionCounts;
  string outputDirectoryName;
  string commandLine;
  string _experimentName;
  
  set< string > _geneUniverse;
  
public:
  
  Reporter()
      : _allPredictions(), _allEvaluations(), _allEvaluationSummaries(),
        _allDetailedCVResults(), predictionResults(),
#ifdef USE_CVRESULT
        cvResults(),
#endif // USE_CVRESULT
        detailedCvResults(),  predictionCounts(),
        outputDirectoryName(),  commandLine(), _experimentName(),
        _geneUniverse()
    {}

  Reporter(string outDir, string cmdLine)
      : _allPredictions(), _allEvaluations(), _allEvaluationSummaries(),
        _allDetailedCVResults(), predictionResults(),
#ifdef USE_CVRESULT
        cvResults(),
#endif // USE_CVRESULT
        detailedCvResults(),  predictionCounts(),
        outputDirectoryName(),  commandLine(), _experimentName(),
        _geneUniverse()
    {
          this->outputDirectoryName = outDir;
          this->commandLine = cmdLine;
    }

  virtual ~Reporter()
    {}

  void addCV(string function, MyGainCVResult result, MyNT threshold,
             string algorithm = "Hopfield");

  // void addCV(string function, int TP, int TN, int FP, int FN, MyNT threshold,
  //            string algorithm = "Hopfield");

  void addCV(string gene, string function, MyGainAnnotationType correctState, 
             MyGainAnnotationType predictedState, MyNT correctStateConfidence, MyNT predictionConfidence,
             string algorithm = "Hopfield");
  

  void addPrediction(string gene, string function, MyNT prob, MyNT input, MyNT threshold,
                     string algorithm = "Hopfield");


  void addPredictionCutBasedConfidence(string gene, string function,
                                       MyNT confidence,
                                       MyNT threshold,
                                       string algorithm = "Hopfield");


  void checkTruePathRuleForPredictions(string algorithm,
                                       MyAnnotations &annotations,
                                       const GeneOntology &go,
                                       ostream &ostr,
                                       map< string, set < string > > &tprViolations);


  void comparePredictions(string algo1, string algo2, ostream &ostr);
  
  
  void computePredictionRanks(string algo, MyAnnotations &differentAnnotations,
                              GeneOntology &go);

  void evaluatePredictions(string algo, MyAnnotations &currentAnnotations,
                           MyAnnotations &newAnnotations, GeneOntology &go);


  void evaluatePredictionsForROCCurvesUsingRanks(string algo, MyAnnotations &differentAnnotations,
                                       GeneOntology &go);
  
  
  void getAlgorithms(set< string > &algorithms);
 
  void getGenesWithPredictions(string algo, set< string > &genes);
  
/*  /// \param[in] algo1, the name of the first algorithm to compare.
  */
  
  /*  /// \note You only have to invoke the method once for each pair of
  */
  void printComparisonPredictionEvaluationROCCurves(const set< string > &algorithms, string outputDir,
                                                    MyAnnotations &annotations, const GeneOntology &go);
  void printComparisonPredictionEvaluationAUCScatterPlots(const set< string > &algorithms, string outputDir, MyAnnotations &annotations);
  
  void printDetailedCVResults(ostream &dcvfstr, bool flush = 1);

  void printCVResults(ostream &cvfstr, bool flush = 1, const BioFunction *functionToPrint = NULL);
  
  
  void printPredictions(ostream &predfstr, int numPredictionsToPrintPerFunction, bool flush = 1, const BioFunction *functionToPrint = NULL);

  void printPredictionEvaluationROCCurves(ostream &ostr);
  
  void printPredictionEvaluations(ostream &ostr, GeneOntology *go = NULL);

  void printPredictionEvaluationSummary(ostream &ostr);

  void readGeneUniverse(string guFile, set< string > &universe);

  
  void readPredictions(string predFile, set< string > *onlyFunctions = NULL,
                       string convertFunction = "");
  
  void readDetailedCVResults(string predFile,
                             const set< string > *onlyFunctions = NULL,
                             string convertFunction = "");
  
  void setExperimentName(string dataset)
    {
      _experimentName = dataset;
    }
    
#ifdef USE_CVRESULT
  void printGroupedCVResults(ostream &cvfstr, const map< string, map< unsigned int, set< GOFunction * > > > &groupedFunctions);
#endif // USE_CVRESULT

  void printCurveDataFromCV(ostream &out, GeneOntology &go, set< BioFunction > functions = set< BioFunction >(), string extra = "");

  void printECWeightedCurveDataFromCV(ostream &out, GeneOntology &go, MyAnnotations &annotations, set< BioFunction > functions = set< BioFunction >(), string extra = "");

  // TODO is this a better way to go than the "flush" option in print methods?
  void clear();


private:


  // fill in a number of details for algoSummary. this process started
  // at the end of Reporter::evaluatePredictions().
  void _computeAlgorithmEvalationSummary(string algo,
                                         PredictionEvaluationSummary &algoSummary);
  
  // compute various confidence related histograms.
  void _computeConfidenceDistributions(string algo,
                                       PredictionEvaluationSummary &summary);
                                       
  void _computeRankDistributions(string algo,
                                       PredictionEvaluationSummary &summary);
};



#endif // _REPORTER_H