Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add timing output to the end of ngen runs #696

Merged
merged 4 commits into from
Jan 8, 2024
Merged

Add timing output to the end of ngen runs #696

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
afa7de5
Refactor end-of-run MPI/Routing logic
PhilMiller Jan 3, 2024
6c8cb15
Add coarse top-level timing profile output
PhilMiller Jan 3, 2024
66721d4
Declare mpi_rank unconditionally to simplify other logic
PhilMiller Jan 5, 2024
7dc6c12
Take advantage of always defining mpi_rank
PhilMiller Jan 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 46 additions & 21 deletions src/NGen.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
#include <fstream>
#include <string>
#include <unordered_map>
#include <chrono>

#include <utilities/span.hpp>

Expand Down Expand Up @@ -37,6 +38,9 @@ std::string nexusDataFile = "";
std::string REALIZATION_CONFIG_PATH = "";
bool is_subdivided_hydrofabric_wanted = false;

// Define in the non-MPI case so that we don't need to conditionally compile `if (mpi_rank == 0)`
int mpi_rank = 0;

#ifdef NGEN_MPI_ACTIVE

#ifndef MPI_HF_SUB_CLI_FLAG
Expand All @@ -49,9 +53,8 @@ bool is_subdivided_hydrofabric_wanted = false;
#include <HY_Features_MPI.hpp>

std::string PARTITION_PATH = "";
int mpi_rank;
int mpi_num_procs;
#endif
#endif // NGEN_MPI_ACTIVE

PhilMiller marked this conversation as resolved.
Show resolved Hide resolved
#include <Layer.hpp>
#include <SurfaceLayer.hpp>
Expand Down Expand Up @@ -136,22 +139,25 @@ int main(int argc, char *argv[]) {
MPI_Init(nullptr, nullptr);
MPI_Comm_rank(MPI_COMM_WORLD, &mpi_rank);
MPI_Comm_size(MPI_COMM_WORLD, &mpi_num_procs);
#endif

if (mpi_rank == 0)
#endif
{
std::ostringstream output;
output << ngen::exec_info::build_summary;
ngen::exec_info::runtime_summary(output);
std::cout << output.str() << std::endl;
} // if (mpi_rank == 0)

#if NGEN_WITH_MPI
MPI_Finalize();
#endif

exit(1);
}

auto time_start = std::chrono::steady_clock::now();

std::cout << "NGen Framework " << ngen_VERSION_MAJOR << "."
<< ngen_VERSION_MINOR << "."
<< ngen_VERSION_PATCH << std::endl;
Expand Down Expand Up @@ -388,11 +394,8 @@ int main(int argc, char *argv[]) {
//realization collection is created
#ifdef NGEN_ROUTING_ACTIVE
std::unique_ptr<routing_py_adapter::Routing_Py_Adapter> router;
#ifdef NGEN_MPI_ACTIVE
//If rank == 0, do routing
if( mpi_rank == 0 )
{ // Run t-route from single process
#endif //NGEN_MPI_ACTIVE
if(manager->get_using_routing()) {
std::cout<<"Using Routing"<<std::endl;
std::string t_route_config_file_with_path = manager->get_t_route_config_file_with_path();
Expand All @@ -401,9 +404,7 @@ int main(int argc, char *argv[]) {
else {
std::cout<<"Not Using Routing"<<std::endl;
}
#ifdef NGEN_MPI_ACTIVE
}
#endif //NGEN_MPI_ACTIVE
#endif //NGEN_ROUTING_ACTIVE
std::cout<<"Building Feature Index\n";
std::string link_key = "toid";
Expand Down Expand Up @@ -483,6 +484,9 @@ int main(int argc, char *argv[]) {

}

auto time_done_init = std::chrono::steady_clock::now();
std::chrono::duration<double> time_elapsed_init = time_done_init - time_start;

//Now loop some time, iterate catchments, do stuff for total number of output times
auto num_times = manager->Simulation_Time_Object->get_total_output_times();
for( int count = 0; count < num_times; count++)
Expand Down Expand Up @@ -534,18 +538,26 @@ int main(int argc, char *argv[]) {
}

} //done time
std::cout<<"Finished "<<manager->Simulation_Time_Object->get_total_output_times()<<" timesteps."<<std::endl;

#if NGEN_MPI_ACTIVE
MPI_Barrier(MPI_COMM_WORLD);
#endif

if (mpi_rank == 0)
{
std::cout << "Finished " << manager->Simulation_Time_Object->get_total_output_times() << " timesteps." << std::endl;
}

#ifdef NGEN_ROUTING_ACTIVE
auto time_done_simulation = std::chrono::steady_clock::now();
std::chrono::duration<double> time_elapsed_simulation = time_done_simulation - time_done_init;

#ifdef NGEN_MPI_ACTIVE
//Syncronization here. MPI barrier. If rank == 0, do routing
#ifdef NGEN_MPI_ACTIVE
MPI_Barrier(MPI_COMM_WORLD);
MPI_Finalize();
if( mpi_rank == 0 )
#endif

#ifdef NGEN_ROUTING_ACTIVE
if (mpi_rank == 0)
{ // Run t-route from single process
#endif //NGEN_MPI_ACTIVE
if(manager->get_using_routing()) {
//Note: Currently, delta_time is set in the t-route yaml configuration file, and the
//number_of_timesteps is determined from the total number of nexus outputs in t-route.
Expand All @@ -557,13 +569,26 @@ int main(int argc, char *argv[]) {

router->route(number_of_timesteps, delta_time);
}
#ifdef NGEN_MPI_ACTIVE
}
#endif //NGEN_MPI_ACTIVE
#else
#ifdef NGEN_MPI_ACTIVE
#endif

auto time_done_routing = std::chrono::steady_clock::now();
std::chrono::duration<double> time_elapsed_routing = time_done_routing - time_done_simulation;

if (mpi_rank == 0)
{
std::cout << "NGen top-level timings:"
<< "\n\tNGen::init: " << time_elapsed_init.count()
<< "\n\tNGen::simulation: " << time_elapsed_simulation.count()
#ifdef NGEN_ROUTING_ACTIVE
<< "\n\tNGen::routing: " << time_elapsed_routing.count()
#endif
<< std::endl;
}

#ifdef NGEN_MPI_ACTIVE
MPI_Finalize();
#endif //NGEN_MPI_ACTIVE
#endif // NGEN_ROUTING_ACTIVE
#endif

return 0;
}