Merge pull request #151 from davidsd/minor-fixes

Minor fixes: more timers and output messages, lightweight git tags, use total time for --maxRuntime

.circleci/config.yml

@@ -114,14 +114,14 @@ workflows:
       - build:
           filters:
             tags:
-              only: /.*/
+              only: /^\d+\.\d+\.\d+$/
 
       - test:
           requires:
             - build
           filters:
             tags:
-              only: /.*/
+              only: /^\d+\.\d+\.\d+$/
 
       - deploy-master:
           filters:
@@ -134,7 +134,7 @@ workflows:
       - deploy-tag:
           filters:
             tags:
-              only: /.*/ # All tags
+              only: /^\d+\.\d+\.\d+$/ # only release tags, e.g 2.6.0
             # See https://discuss.circleci.com/t/tag-not-triggered-by-circleci-not-filter-in-workflow/32036
             branches:
               ignore: /.*/ # Ignore all branches, otherwise the job is triggered by any git push

src/Timers.hxx

@@ -237,6 +237,8 @@ struct Scoped_Timer : boost::noncopyable
     timers.prefix = old_prefix;
   }
 
+  [[nodiscard]] const Timer &timer() const { return my_timer; }
+
 private:
   Timers &timers;
   Timer &my_timer;

src/outer_limits/compute_optimal/compute_optimal.cxx

@@ -61,7 +61,8 @@ std::vector<El::BigFloat> compute_optimal(
   const std::vector<std::vector<El::BigFloat>> &initial_points,
   const std::vector<El::BigFloat> &objectives,
   const std::vector<El::BigFloat> &normalization,
-  const Outer_Parameters &parameters_in)
+  const Outer_Parameters &parameters_in,
+  const std::chrono::time_point<std::chrono::high_resolution_clock> &start_time)
 {
   if(initial_points.size() != function_blocks.size())
     {
@@ -213,9 +214,9 @@ std::vector<El::BigFloat> compute_optimal(
             }
 
           Timers timers(parameters.verbosity >= Verbosity::debug);
-          SDP_Solver_Terminate_Reason reason
-            = solver.run(parameters.solver, parameters.verbosity,
-                         parameter_properties, block_info, sdp, grid, timers);
+          SDP_Solver_Terminate_Reason reason = solver.run(
+            parameters.solver, parameters.verbosity, parameter_properties,
+            block_info, sdp, grid, start_time, timers);
 
           for(size_t index(0); index < block_info.block_indices.size();
               ++index)

src/outer_limits/main.cxx

@@ -67,7 +67,9 @@ std::vector<El::BigFloat> compute_optimal(
   const std::vector<std::vector<El::BigFloat>> &initial_points,
   const std::vector<El::BigFloat> &objectives,
   const std::vector<El::BigFloat> &normalization,
-  const Outer_Parameters &parameters_in);
+  const Outer_Parameters &parameters_in,
+  const std::chrono::time_point<std::chrono::high_resolution_clock>
+    &start_time);
 
 int main(int argc, char **argv)
 {
@@ -83,13 +85,13 @@ int main(int argc, char **argv)
   // El::gmp wants base-2 bits, but boost::multiprecision wants
   // base-10 digits.
   Boost_Float::default_precision(precision * log(2) / log(10));
-
+  auto start_time = std::chrono::high_resolution_clock::now();
   if(parameters.verbosity >= Verbosity::regular && El::mpi::Rank() == 0)
     {
-      std::cout << "Outer_Limits started at "
-                << boost::posix_time::second_clock::local_time() << '\n'
-                << parameters << '\n'
-                << std::flush;
+      std::cout << boost::posix_time::second_clock::local_time()
+                << " Start Outer_Limits" << '\n'
+                << "Version: " << SDPB_VERSION_STRING << '\n'
+                << parameters << std::endl;
     }
 
   std::vector<El::BigFloat> objectives, normalization;
@@ -100,8 +102,9 @@ int main(int argc, char **argv)
   std::vector<std::vector<El::BigFloat>> initial_points;
   read_points(parameters.points_path, initial_points);
 
-  std::vector<El::BigFloat> weights(compute_optimal(
-    functions, initial_points, objectives, normalization, parameters));
+  std::vector<El::BigFloat> weights(compute_optimal(functions, initial_points,
+                                                    objectives, normalization,
+                                                    parameters, start_time));
 
   El::BigFloat optimal(0);
   for(size_t index(0); index < objectives.size(); ++index)

src/pvm2sdp/main.cxx

@@ -53,9 +53,11 @@ int main(int argc, char **argv)
           throw std::runtime_error("Output file '" + output_path.string()
                                    + "' exists and is a directory");
         }
+      Timers timers(debug);
       write_sdpb_input_files(output_path, output_format, rank, num_blocks,
                              command_arguments, objective_const,
-                             dual_objective_b, dual_constraint_groups, debug);
+                             dual_objective_b, dual_constraint_groups, timers,
+                             debug);
     }
   catch(std::exception &e)
     {

src/sdp2input/main.cxx

@@ -118,6 +118,12 @@ int main(int argc, char **argv)
         }
       write_output(output_path, output_format, command_arguments, objectives,
                    normalization, matrices, timers, debug);
+      if(El::mpi::Rank() == 0)
+        {
+          El::Output("Processed ", matrices.size(), " SDP blocks in ",
+                     (double)timer.timer().elapsed_milliseconds() / 1000,
+                     " seconds, output: ", output_path.string());
+        }
       if(debug)
         {
           timers.write_profile(output_path.string() + ".profiling/profiling."

src/sdp2input/write_output/write_output.cxx

@@ -135,8 +135,7 @@ void write_output(const fs::path &output_path, Block_File_Format output_format,
     }
   matrices_timer.stop();
   convert_timer.stop();
-  Scoped_Timer write_timer(timers, "write");
   write_sdpb_input_files(output_path, output_format, rank, matrices.size(),
                          command_arguments, objective_const, dual_objective_b,
-                         dual_constraint_groups, debug);
+                         dual_constraint_groups, timers, debug);
 }

src/sdp_convert.hxx

@@ -4,6 +4,7 @@
 #include "sdp_convert/Block_File_Format.hxx"
 #include "sdp_convert/Dual_Constraint_Group.hxx"
 #include "sdp_convert/write_vector.hxx"
+#include "Timers.hxx"
 
 #include <filesystem>
 
@@ -16,4 +17,4 @@ void write_sdpb_input_files(
   const El::BigFloat &objective_const,
   const std::vector<El::BigFloat> &dual_objective_b,
   const std::vector<Dual_Constraint_Group> &dual_constraint_groups,
-  bool debug);
+  Timers &timers, const bool debug);

src/sdp_convert/write_sdpb_input_files.cxx

@@ -2,6 +2,7 @@
 #include "byte_counter.hxx"
 #include "Archive_Writer.hxx"
 #include "../sdp_convert.hxx"
+#include "../Timers.hxx"
 
 #include <filesystem>
 
@@ -64,7 +65,8 @@ namespace
     return temp_dir / El::BuildString("block_info_", block_index, ".json");
   }
 
-  fs::path get_block_data_path(const fs::path &temp_dir, size_t block_index, Block_File_Format format)
+  fs::path get_block_data_path(const fs::path &temp_dir, size_t block_index,
+                               Block_File_Format format)
   {
     std::string name = El::BuildString("block_data_", block_index);
     switch(format)
@@ -76,7 +78,8 @@ namespace
     return temp_dir / name;
   }
 
-  void archive_gzipped_file(const fs::path &path, const int64_t &num_bytes, Archive_Writer &writer)
+  void archive_gzipped_file(const fs::path &path, const int64_t &num_bytes,
+                            Archive_Writer &writer)
   {
     boost::iostreams::filtering_stream<boost::iostreams::input> input_stream;
     input_stream.push(boost::iostreams::gzip_decompressor());
@@ -93,8 +96,10 @@ void write_sdpb_input_files(
   const El::BigFloat &objective_const,
   const std::vector<El::BigFloat> &dual_objective_b,
   const std::vector<Dual_Constraint_Group> &dual_constraint_groups,
-  const bool debug)
+  Timers &timers, const bool debug)
 {
+  Scoped_Timer write_timer(timers, "write_sdp");
+
   fs::path temp_dir(output_path);
   temp_dir += "_temp";
   fs::create_directories(temp_dir);
@@ -104,35 +109,50 @@ void write_sdpb_input_files(
   std::vector<size_t> block_info_sizes(num_blocks, 0);
   std::vector<size_t> block_data_sizes(num_blocks, 0);
 
-  for(auto &group : dual_constraint_groups)
-    {
-      size_t width = group.constraint_matrix.Width();
-      size_t dual_objective_size = dual_objective_b.size();
-      if(width != dual_objective_size)
+  {
+    Scoped_Timer block_files_timer(timers, "block_files");
+    for(auto &group : dual_constraint_groups)
+      {
+        size_t width = group.constraint_matrix.Width();
+        size_t dual_objective_size = dual_objective_b.size();
+        if(width != dual_objective_size)
+          {
+            El::RuntimeError(" Block width=", width,
+                             " and dual objective size=", dual_objective_size,
+                             " should be equal.");
+          }
+
         {
-          El::RuntimeError(" Block width=", width,
-                           " and dual objective size=", dual_objective_size,
-                           " should be equal.");
+          Scoped_Timer block_info_timer(
+            timers, "block_info_" + std::to_string(group.block_index));
+          const auto block_info_path
+            = get_block_info_path(temp_dir, group.block_index);
+          block_info_sizes.at(group.block_index) = write_data_and_count_bytes(
+            block_info_path,
+            [&](std::ostream &os) { write_block_info_json(os, group); });
         }
 
-      const auto block_info_path
-        = get_block_info_path(temp_dir, group.block_index);
-      block_info_sizes.at(group.block_index)
-        = write_data_and_count_bytes(block_info_path, [&](std::ostream &os) {
-            write_block_info_json(os, group);
-          });
-
-      const auto block_data_path
-        = get_block_data_path(temp_dir, group.block_index, output_format);
-      block_data_sizes.at(group.block_index) = write_data_and_count_bytes(
-        block_data_path,
-        [&](std::ostream &os) { write_block_data(os, group, output_format); },
-        output_format == bin);
-    }
-  El::mpi::Reduce(block_info_sizes.data(), block_info_sizes.size(),
-                  El::mpi::SUM, 0, El::mpi::COMM_WORLD);
-  El::mpi::Reduce(block_data_sizes.data(), block_data_sizes.size(),
-                  El::mpi::SUM, 0, El::mpi::COMM_WORLD);
+        {
+          Scoped_Timer block_data_timer(
+            timers, "block_data_" + std::to_string(group.block_index));
+          const auto block_data_path
+            = get_block_data_path(temp_dir, group.block_index, output_format);
+          block_data_sizes.at(group.block_index) = write_data_and_count_bytes(
+            block_data_path,
+            [&](std::ostream &os) {
+              write_block_data(os, group, output_format);
+            },
+            output_format == bin);
+        }
+      }
+  }
+  {
+    Scoped_Timer reduce_timer(timers, "mpi_reduce_block_sizes");
+    El::mpi::Reduce(block_info_sizes.data(), block_info_sizes.size(),
+                    El::mpi::SUM, 0, El::mpi::COMM_WORLD);
+    El::mpi::Reduce(block_data_sizes.data(), block_data_sizes.size(),
+                    El::mpi::SUM, 0, El::mpi::COMM_WORLD);
+  }
   if(debug)
     {
       print_matrix_sizes(rank, dual_objective_b, dual_constraint_groups);
@@ -152,34 +172,50 @@ void write_sdpb_input_files(
           });
 
       // write all files to sdp.zip archive
+      Scoped_Timer zip_timer(timers, "zip");
       Archive_Writer writer(output_path);
       // control.json
-      archive_gzipped_file(control_path, num_control_bytes, writer);
-      fs::remove(control_path);
+      {
+        Scoped_Timer control_timer(timers, "control");
+        archive_gzipped_file(control_path, num_control_bytes, writer);
+        fs::remove(control_path);
+      }
       // objectives.json
-      archive_gzipped_file(objectives_path, num_objectives_bytes, writer);
-      fs::remove(objectives_path);
+      {
+        Scoped_Timer objectives_timer(timers, "objectives");
+        archive_gzipped_file(objectives_path, num_objectives_bytes, writer);
+        fs::remove(objectives_path);
+      }
 
       // block_info_XXX.json
-      // We add block_info before all block_data,
-      // so that SDPB can read them fast and skip the rest of archive
-      for(size_t block_index = 0; block_index != num_blocks; ++block_index)
-        {
-          const auto block_info_path
-            = get_block_info_path(temp_dir, block_index);
-          archive_gzipped_file(block_info_path,
-                               block_info_sizes.at(block_index), writer);
-          fs::remove(block_info_path);
-        }
+      {
+        Scoped_Timer block_info_timer(timers, "block_info");
+
+        // We add block_info before all block_data,
+        // so that SDPB can read them fast and skip the rest of archive
+        for(size_t block_index = 0; block_index != num_blocks; ++block_index)
+          {
+            Scoped_Timer index_timer(timers, std::to_string(block_index));
+            const auto block_info_path
+              = get_block_info_path(temp_dir, block_index);
+            archive_gzipped_file(block_info_path,
+                                 block_info_sizes.at(block_index), writer);
+            fs::remove(block_info_path);
+          }
+      }
       // block_data_XXX.bin (or .json)
-      for(size_t block_index = 0; block_index != num_blocks; ++block_index)
-        {
-          const auto block_data_path
-            = get_block_data_path(temp_dir, block_index, output_format);
-          archive_gzipped_file(block_data_path,
-                               block_data_sizes.at(block_index), writer);
-          fs::remove(block_data_path);
-        }
+      {
+        Scoped_Timer block_data_timer(timers, "block_data");
+        for(size_t block_index = 0; block_index != num_blocks; ++block_index)
+          {
+            Scoped_Timer index_timer(timers, std::to_string(block_index));
+            const auto block_data_path
+              = get_block_data_path(temp_dir, block_index, output_format);
+            archive_gzipped_file(block_data_path,
+                                 block_data_sizes.at(block_index), writer);
+            fs::remove(block_data_path);
+          }
+      }
       // Do not call remove_all() to ensure that we
       // don't remove anything useful.
       // This function will fail if temp_dir is not empty.

src/sdp_solve/SDP/SDP/read_blocks/read_blocks.cxx

@@ -29,6 +29,11 @@ void read_block_stream(
 void read_blocks(const fs::path &sdp_path, const El::Grid &grid,
                  const Block_Info &block_info, SDP &sdp)
 {
+  if(!fs::exists(sdp_path))
+    {
+      El::RuntimeError("SDP path '" + sdp_path.string() + "' does not exist");
+    }
+
   const size_t num_blocks(block_info.block_indices.size());
   sdp.primal_objective_c.blocks.resize(num_blocks);
   sdp.free_var_matrix.blocks.resize(num_blocks);

src/sdp_solve/SDP/SDP/read_objectives.cxx

@@ -38,6 +38,11 @@ namespace
 void read_objectives(const fs::path &sdp_path, const El::Grid &grid, El::BigFloat &objective_const,
                      El::DistMatrix<El::BigFloat> &dual_objective_b)
 {
+  if(!fs::exists(sdp_path))
+    {
+      El::RuntimeError("SDP path '" + sdp_path.string() + "' does not exist");
+    }
+
   const std::string objectives_name("objectives.json");
   if(fs::is_regular_file(sdp_path))
     {

src/sdp_solve/SDP_Solver.hxx

@@ -80,10 +80,11 @@ public:
              const size_t &dual_objective_b_height);
 
   SDP_Solver_Terminate_Reason
-  run(const Solver_Parameters &parameters,
-      const Verbosity &verbosity,
+  run(const Solver_Parameters &parameters, const Verbosity &verbosity,
       const boost::property_tree::ptree &parameter_properties,
       const Block_Info &block_info, const SDP &sdp, const El::Grid &grid,
+      const std::chrono::time_point<std::chrono::high_resolution_clock>
+        &start_time,
       Timers &timers);
 
   void step(