support turning off verbose logging

git-svn-id: https://ws10smt.googlecode.com/svn/trunk@652 ec762483-ff6d-05da-a07a-a48fb63a330f

decoder/apply_models.cc

@@ -13,6 +13,7 @@
 
 #include <boost/functional/hash.hpp>
 
+#include "verbose.h"
 #include "hg.h"
 #include "ff.h"
 
@@ -170,7 +171,7 @@ class CubePruningRescorer {
       out(*o),
       D(in.nodes_.size()),
       pop_limit_(pop_limit) {
-    cerr << "  Applying feature functions (cube pruning, pop_limit = " << pop_limit_ << ')' << endl;
+    if (!SILENT) cerr << "  Applying feature functions (cube pruning, pop_limit = " << pop_limit_ << ')' << endl;
     node_states_.reserve(kRESERVE_NUM_NODES);
   }
 
@@ -181,14 +182,16 @@ class CubePruningRescorer {
     int every = 1;
     if (num_nodes > 100) every = 10;
     assert(in.nodes_[pregoal].out_edges_.size() == 1);
-    cerr << "    ";
+    if (!SILENT) cerr << "    ";
     for (int i = 0; i < in.nodes_.size(); ++i) {
-      if (i % every == 0) cerr << '.';
+      if (!SILENT && i % every == 0) cerr << '.';
       KBest(i, i == goal_id);
     }
-    cerr << endl;
-    cerr << "  Best path: " << log(D[goal_id].front()->vit_prob_)
-         << "\t" << log(D[goal_id].front()->est_prob_) << endl;
+    if (!SILENT) {
+      cerr << endl;
+      cerr << "  Best path: " << log(D[goal_id].front()->vit_prob_)
+           << "\t" << log(D[goal_id].front()->est_prob_) << endl;
+    }
     out.PruneUnreachable(D[goal_id].front()->node_index_);
     FreeAll();
   }
@@ -379,12 +382,12 @@ struct NoPruningRescorer {
     int every = 1;
     if (num_nodes > 100) every = 10;
     assert(in.nodes_[pregoal].out_edges_.size() == 1);
-    cerr << "    ";
+    if (!SILENT) cerr << "    ";
     for (int i = 0; i < in.nodes_.size(); ++i) {
-      if (i % every == 0) cerr << '.';
+      if (!SILENT && i % every == 0) cerr << '.';
       ProcessOneNode(i, i == goal_id);
     }
-    cerr << endl;
+    if (!SILENT) cerr << endl;
   }
 
  private:

decoder/bottom_up_parser.cc

@@ -10,14 +10,15 @@
 #include "hg.h"
 #include "array2d.h"
 #include "tdict.h"
+#include "verbose.h"
 
 using namespace std;
 
 struct ParserStats {
   ParserStats() : active_items(), passive_items() {}
   void Reset() { active_items=0; passive_items=0; }
   void Report() {
-    cerr << "  ACTIVE ITEMS: " << active_items << "\tPASSIVE ITEMS: " << passive_items << endl;
+    if (!SILENT) cerr << "  ACTIVE ITEMS: " << active_items << "\tPASSIVE ITEMS: " << passive_items << endl;
   }
   int active_items;
   int passive_items;
@@ -172,7 +173,7 @@ PassiveChart::PassiveChart(const string& goal,
   for (int i = 0; i < grammars_.size(); ++i)
     act_chart_[i] = new ActiveChart(forest, *this);
   if (!kGOAL) kGOAL = TD::Convert("Goal") * -1;
-  cerr << "  Goal category: [" << goal << ']' << endl;
+  if (!SILENT) cerr << "  Goal category: [" << goal << ']' << endl;
 }
 
 void PassiveChart::ApplyRule(const int i,
@@ -241,9 +242,9 @@ bool PassiveChart::Parse() {
   for (int gi = 0; gi < grammars_.size(); ++gi)
     act_chart_[gi]->SeedActiveChart(*grammars_[gi]);
 
-  cerr << "    ";
+  if (!SILENT) cerr << "    ";
   for (int l=1; l<input_.size()+1; ++l) {
-    cerr << '.';
+    if (!SILENT) cerr << '.';
     for (int i=0; i<input_.size() + 1 - l; ++i) {
       int j = i + l;
       for (int gi = 0; gi < grammars_.size(); ++gi) {
@@ -278,7 +279,7 @@ bool PassiveChart::Parse() {
       }
     }
   }
-  cerr << endl;
+  if (!SILENT) cerr << endl;
 
   if (GoalFound())
     forest_->PruneUnreachable(forest_->nodes_.size() - 1);

decoder/decoder.cc

@@ -10,6 +10,7 @@
 #include "filelib.h"
 #include "fdict.h"
 #include "timing_stats.h"
+#include "verbose.h"
 
 #include "translator.h"
 #include "phrasebased_translator.h"
@@ -51,6 +52,8 @@ static bool verbose_feature_functions=true;
 namespace Hack { void MaxTrans(const Hypergraph& in, int beam_size); }
 namespace NgramCache { void Clear(); }
 
+DecoderObserver::~DecoderObserver() {}
+void DecoderObserver::NotifyDecodingStart(const SentenceMetadata& smeta) {}
 void DecoderObserver::NotifySourceParseFailure(const SentenceMetadata&) {}
 void DecoderObserver::NotifyTranslationForest(const SentenceMetadata&, Hypergraph*) {}
 void DecoderObserver::NotifyAlignmentFailure(const SentenceMetadata&) {}
@@ -639,22 +642,25 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
   if (sgml.find("id") != sgml.end())
     sent_id = atoi(sgml["id"].c_str());
 
-  cerr << "\nINPUT: ";
-  if (buf.size() < 100)
-    cerr << buf << endl;
-  else {
-    size_t x = buf.rfind(" ", 100);
-    if (x == string::npos) x = 100;
-    cerr << buf.substr(0, x) << " ..." << endl;
+  if (!SILENT) {
+    cerr << "\nINPUT: ";
+    if (buf.size() < 100)
+      cerr << buf << endl;
+    else {
+      size_t x = buf.rfind(" ", 100);
+      if (x == string::npos) x = 100;
+      cerr << buf.substr(0, x) << " ..." << endl;
+    }
+    cerr << "  id = " << sent_id << endl;
   }
-  cerr << "  id = " << sent_id << endl;
   string to_translate;
   Lattice ref;
   ParseTranslatorInputLattice(buf, &to_translate, &ref);
   const unsigned srclen=NTokens(to_translate,' ');
 //FIXME: should get the avg. or max source length of the input lattice (like Lattice::dist_(start,end)); but this is only used to scale beam parameters (optionally) anyway so fidelity isn't important.
   const bool has_ref = ref.size() > 0;
   SentenceMetadata smeta(sent_id, ref);
+  o->NotifyDecodingStart(smeta);
   Hypergraph forest;          // -LM forest
   translator->ProcessMarkupHints(sgml);
   Timer t("Translation");
@@ -664,15 +670,15 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
   translator->SentenceComplete();
 
   if (!translation_successful) {
-    cerr << "  NO PARSE FOUND.\n";
+    if (!SILENT) cerr << "  NO PARSE FOUND.\n";
     o->NotifySourceParseFailure(smeta);
     o->NotifyDecodingComplete(smeta);
     return false;
   }
 
   const bool show_tree_structure=conf.count("show_tree_structure");
   const bool show_features=conf.count("show_features");
-  forest_stats(forest,"  -LM forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
+  if (!SILENT) forest_stats(forest,"  -LM forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
   if (conf.count("show_expected_length")) {
     const PRPair<double, double> res =
       Inside<PRPair<double, double>,
@@ -714,7 +720,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
                   &lm_forest);
     forest.swap(lm_forest);
     forest.Reweight(feature_weights);
-    forest_stats(forest,"  +LM forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
+    if (!SILENT) forest_stats(forest,"  +LM forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
   }
 
   maybe_prune(forest,conf,"beam_prune","density_prune","+LM",srclen);
@@ -729,30 +735,30 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
     Hypergraph fsa_forest;
     assert(fsa_ffs.size()==1);
     ApplyFsaBy cfg(str("apply_fsa_by",conf),pop_limit);
-    cerr << "FSA rescoring with "<<cfg<<" "<<fsa_ffs[0]->describe()<<endl;
+    if (!SILENT) cerr << "FSA rescoring with "<<cfg<<" "<<fsa_ffs[0]->describe()<<endl;
     ApplyFsaModels(hgcfg,smeta,*fsa_ffs[0],feature_weights,cfg,&fsa_forest);
     forest.swap(fsa_forest);
     forest.Reweight(feature_weights);
-    forest_stats(forest,"  +FSA forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
+    if (!SILENT) forest_stats(forest,"  +FSA forest",show_tree_structure,show_features,feature_weights,oracle.show_derivation);
   }
 
   // Oracle Rescoring
   if(get_oracle_forest) {
     Oracle oc=oracle.ComputeOracle(smeta,&forest,FeatureVector(feature_weights),10,conf["forest_output"].as<std::string>());
-    cerr << "  +Oracle BLEU forest (nodes/edges): " << forest.nodes_.size() << '/' << forest.edges_.size() << endl;
-    cerr << "  +Oracle BLEU (paths): " << forest.NumberOfPaths() << endl;
+    if (!SILENT) cerr << "  +Oracle BLEU forest (nodes/edges): " << forest.nodes_.size() << '/' << forest.edges_.size() << endl;
+    if (!SILENT) cerr << "  +Oracle BLEU (paths): " << forest.NumberOfPaths() << endl;
     oc.hope.Print(cerr,"  +Oracle BLEU");
     oc.fear.Print(cerr,"  -Oracle BLEU");
     //Add 1-best translation (trans) to psuedo-doc vectors
-    oracle.IncludeLastScore(&cerr);
+    if (!SILENT) oracle.IncludeLastScore(&cerr);
   }
   o->NotifyTranslationForest(smeta, &forest);
 
   // TODO I think this should probably be handled by an Observer
   if (conf.count("forest_output") && !has_ref) {
     ForestWriter writer(str("forest_output",conf), sent_id);
     if (FileExists(writer.fname_)) {
-      cerr << "  Unioning...\n";
+      if (!SILENT) cerr << "  Unioning...\n";
       Hypergraph new_hg;
       {
         ReadFile rf(writer.fname_);
@@ -812,15 +818,15 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
     HypergraphIO::WriteAsCFG(forest);
   if (has_ref) {
     if (HG::Intersect(ref, &forest)) {
-      cerr << "  Constr. forest (nodes/edges): " << forest.nodes_.size() << '/' << forest.edges_.size() << endl;
-      cerr << "  Constr. forest       (paths): " << forest.NumberOfPaths() << endl;
+      if (!SILENT) cerr << "  Constr. forest (nodes/edges): " << forest.nodes_.size() << '/' << forest.edges_.size() << endl;
+      if (!SILENT) cerr << "  Constr. forest       (paths): " << forest.NumberOfPaths() << endl;
       if (crf_uniform_empirical) {
-        cerr << "  USING UNIFORM WEIGHTS\n";
+        if (!SILENT) cerr << "  USING UNIFORM WEIGHTS\n";
         for (int i = 0; i < forest.edges_.size(); ++i)
           forest.edges_[i].edge_prob_=prob_t::One();
       } else {
         forest.Reweight(feature_weights);
-        cerr << "  Constr. VitTree: " << ViterbiFTree(forest) << endl;
+        if (!SILENT) cerr << "  Constr. VitTree: " << ViterbiFTree(forest) << endl;
       }
       if (conf.count("show_partition")) {
          const prob_t z = Inside<prob_t, EdgeProb>(forest);
@@ -830,7 +836,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
       if (conf.count("forest_output")) {
         ForestWriter writer(str("forest_output",conf), sent_id);
         if (FileExists(writer.fname_)) {
-          cerr << "  Unioning...\n";
+          if (!SILENT) cerr << "  Unioning...\n";
           Hypergraph new_hg;
           {
             ReadFile rf(writer.fname_);
@@ -893,7 +899,7 @@ bool DecoderImpl::Decode(const string& input, DecoderObserver* o) {
       if (conf.count("graphviz")) forest.PrintGraphviz();
     } else {
       o->NotifyAlignmentFailure(smeta);
-      cerr << "  REFERENCE UNREACHABLE.\n";
+      if (!SILENT) cerr << "  REFERENCE UNREACHABLE.\n";
       if (write_gradient) {
         cout << endl << flush;
       }

decoder/decoder.h

@@ -12,6 +12,8 @@ struct Hypergraph;
 struct DecoderImpl;
 
 struct DecoderObserver {
+  virtual ~DecoderObserver();
+  virtual void NotifyDecodingStart(const SentenceMetadata& smeta);
   virtual void NotifySourceParseFailure(const SentenceMetadata& smeta);
   virtual void NotifyTranslationForest(const SentenceMetadata& smeta, Hypergraph* hg);
   virtual void NotifyAlignmentFailure(const SentenceMetadata& semta);

decoder/hg_intersect.cc

@@ -5,6 +5,7 @@
 #include "fast_lexical_cast.hpp"
 #include <boost/functional/hash.hpp>
 
+#include "verbose.h"
 #include "tdict.h"
 #include "hg.h"
 #include "trule.h"
@@ -50,7 +51,7 @@ struct RuleFilter {
 };
 
 static bool FastLinearIntersect(const Lattice& target, Hypergraph* hg) {
-  cerr << "  Fast linear-chain intersection...\n";
+  if (!SILENT) cerr << "  Fast linear-chain intersection...\n";
   vector<bool> prune(hg->edges_.size(), false);
   set<int> cov;
   map<const TRule*, TRulePtr> inverted_rules;

decoder/inside_outside.h

@@ -1,5 +1,5 @@
-#ifndef _INSIDE_H_
-#define _INSIDE_H_
+#ifndef _INSIDE_OUTSIDE_H_
+#define _INSIDE_OUTSIDE_H_
 
 #include <vector>
 #include <algorithm>

decoder/scfg_translator.cc

@@ -14,6 +14,7 @@
 #include "sentence_metadata.h"
 #include "tdict.h"
 #include "viterbi.h"
+#include "verbose.h"
 
 #define foreach         BOOST_FOREACH
 #define reverse_foreach BOOST_REVERSE_FOREACH
@@ -100,7 +101,7 @@ struct SCFGTranslatorImpl {
     LatticeTools::ConvertTextOrPLF(input, &lattice);
     smeta->SetSourceLength(lattice.size());
     if (add_pass_through_rules){
-      cerr << "Adding pass through grammar" << endl;
+      if (!SILENT) cerr << "Adding pass through grammar" << endl;
       PassThroughGrammar* g = new PassThroughGrammar(lattice, default_nt, ctf_iterations_);
       g->SetGrammarName("PassThrough");
       glist.push_back(GrammarPtr(g));
@@ -109,13 +110,13 @@ struct SCFGTranslatorImpl {
       if(printGrammarsUsed)
         cerr << "Using grammar::" << glist[gi]->GetGrammarName() << endl;
     }
-    cerr << "First pass parse... " << endl;
+    if (!SILENT) cerr << "First pass parse... " << endl;
     ExhaustiveBottomUpParser parser(goal, glist);
     if (!parser.Parse(lattice, forest)){
-      cerr << "parse failed." << endl;
+      if (!SILENT) cerr << "parse failed." << endl;
       return false;
     } else {
-      cerr << "parse succeeded." << endl;
+      if (!SILENT) cerr << "parse succeeded." << endl;
     }
     forest->Reweight(weights);
     if (use_ctf_) {

decoder/sentence_metadata.h

@@ -13,6 +13,8 @@ struct SentenceMetadata {
     trg_len_(ref.size()),
     ref_(has_reference_ ? &ref : NULL) {}
 
+  int GetSentenceId() const { return sent_id_; }
+
   // this should be called by the Translator object after
   // it has parsed the source
   void SetSourceLength(int sl) { src_len_ = sl; }

utils/Makefile.am

@@ -22,6 +22,7 @@ libutils_a_SOURCES = \
   stringlib.cc \
   sparse_vector.cc \
   timing_stats.cc \
+  verbose.cc \
   weights.cc
 
 dict_test_SOURCES = dict_test.cc

utils/timing_stats.cc

@@ -2,6 +2,9 @@
 
 #include <iostream>
 #include "time.h" //cygwin needs
+
+#include "verbose.h"
+
 using namespace std;
 
 map<string, TimerInfo> Timer::stats;
@@ -16,8 +19,10 @@ Timer::~Timer() {
 }
 
 void Timer::Summarize() {
-  for (map<string, TimerInfo>::iterator it = stats.begin(); it != stats.end(); ++it) {
-    cerr << it->first << ": " << it->second.total_time << " secs (" << it->second.calls << " calls)\n";
+  if (!SILENT) {
+    for (map<string, TimerInfo>::iterator it = stats.begin(); it != stats.end(); ++it) {
+      cerr << it->first << ": " << it->second.total_time << " secs (" << it->second.calls << " calls)\n";
+    }
   }
   stats.clear();
 }

utils/verbose.cc

@@ -0,0 +1,4 @@
+#include "verbose.h"
+
+bool SILENT = false;
+

utils/verbose.h

@@ -0,0 +1,8 @@
+#ifndef _VERBOSE_H_
+#define _VERBOSE_H_
+
+extern bool SILENT;
+
+inline void SetSilent(bool s) { SILENT = s; }
+
+#endif