Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

WIP: add SCR to HDF path #126

Open
wants to merge 6 commits into
base: raja
Choose a base branch
from
Open

WIP: add SCR to HDF path #126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
8c59031
add SCR to HDF path
adammoody Feb 11, 2022
461a433
add SCR to getDt for HDF
adammoody Feb 11, 2022
ad59f47
drop extra have and start restart calls
adammoody Feb 11, 2022
7beb75c
add compute_file_suffix given a string
adammoody Feb 11, 2022
a404216
keep previous SCR code in getDt
adammoody Feb 12, 2022
52bc9d2
add scr to non-hdf path
adammoody Feb 12, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
116 changes: 114 additions & 2 deletions src/CheckPoint.C
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -299,6 +299,10 @@ void CheckPoint::compute_file_suffix(int cycle, std::stringstream& fileSuffix) {
fileSuffix << ".sw4checkpoint";
}

void CheckPoint::compute_file_suffix(const char* cycle, std::stringstream& fileSuffix) {
fileSuffix << mCheckPointFile << "." << cycle << ".sw4checkpoint";
}

//-----------------------------------------------------------------------
void CheckPoint::write_checkpoint(float_sw4 a_time, int a_cycle,
vector<Sarray>& a_Um, vector<Sarray>& a_U,
Expand Down Expand Up @@ -338,9 +342,25 @@ void CheckPoint::write_checkpoint(float_sw4 a_time, int a_cycle,
s << fileSuffix.str();
}

#ifndef SW4_USE_SCR
// Keep track of the number of files, save previous file name, and delete the
// second last.
cycle_checkpoints(s.str());
#else
// SCR can delete older checkpoints,
// e.g., SCR_PREFIX_SIZE=3 to keep a sliding window of the last 3

// Inform SCR that a new checkpoint is starting
std::string cycle_num;
cycle_num << "cycle=" << a_cycle;
SCR_Start_output(cycle_num.str().c_str(), SCR_FLAG_CHECKPOINT);

// Ask SCR for the path to write our checkpoint file
// This could be moved just before each H5FCreate call if need to preserve original file name
char scr_file[SCR_MAX_FILENAME];
SCR_Route_file(s.str().c_str(), scr_file);
s = scr_file;
#endif

// Open file from processor zero and write header.
int hsize;
Expand Down Expand Up @@ -441,6 +461,11 @@ void CheckPoint::write_checkpoint(float_sw4 a_time, int a_cycle,
delete[] doubleField;
}
if (iwrite) close(fid);

#ifdef SW4_USE_SCR
int valid = 1;
SCR_Complete_output(valid);
#endif
} // end write_checkpoint()

//-----------------------------------------------------------------------
Expand All @@ -464,7 +489,22 @@ void CheckPoint::read_checkpoint(float_sw4& a_time, int& a_cycle,
s << mRestartPath << "/";
else if (mEW->getPath() != "./")
s << mEW->getPath();

#ifndef SW4_USE_SCR
s << mRestartFile;
#else
// TODO: need to get cycle_num from earlier call to Have_restart

// TODO: this is not right, but you get the idea...
std::stringstream fileSuffix;
compute_file_suffix(cycle_num, fileSuffix);
s << fileSuffix.str();

// Ask SCR for the path to open our checkpoint file
char scr_file[SCR_MAX_FILENAME];
SCR_Route_file(s.str().c_str(), scr_file);
s = scr_file;
#endif
}

// Open file from processor zero and read header.
Expand Down Expand Up @@ -574,19 +614,50 @@ void CheckPoint::read_checkpoint(float_sw4& a_time, int& a_cycle,
delete[] doubleField;
}
if (iread) close(fid);

#ifdef SW4_USE_SCR
int valid = 1;
SCR_Complete_restart(valid);
#endif
}

//-----------------------------------------------------------------------
float_sw4 CheckPoint::getDt() {
#ifndef SW4_USE_SCR
float_sw4 dt;

#ifdef SW4_USE_SCR
// Get checkpoint name (cycle number) from SCR
int have_restart = 0;
char cycle_num[SCR_MAX_FILENAME];
SCR_Have_restart(&have_restart, cycle_num);
if (! have_restart) {
// We expected SCR to have something to get to this point
abort();
}

SCR_Start_restart(cycle_num);
#endif

if (mEW->getRank() == 0) {
std::stringstream s;
if (mRestartPathSet)
s << mRestartPath << "/";
else if (mEW->getPath() != "./")
s << mEW->getPath();

#ifndef SW4_USE_SCR
s << mRestartFile; // string 's' is the file name including path
#else
// TODO: this is not right, but you get the idea...
std::stringstream fileSuffix;
compute_file_suffix(cycle_num, fileSuffix);
s << fileSuffix.str();

// Ask SCR for the path to open our checkpoint file
char scr_file[SCR_MAX_FILENAME];
SCR_Route_file(s.str().c_str(), scr_file);
s = scr_file;
#endif

if (mUseHDF5) {
#ifdef USE_HDF5
Expand Down Expand Up @@ -620,8 +691,8 @@ float_sw4 CheckPoint::getDt() {
}
MPI_Bcast(&dt, 1, mEW->m_mpifloat, 0, mEW->m_cartesian_communicator);
return dt;
#else

#if 0
int have_restart = 0;
char checkpoint_dir[SCR_MAX_FILENAME];
SCR_Have_restart(&have_restart, checkpoint_dir);
Expand Down Expand Up @@ -1119,9 +1190,24 @@ void CheckPoint::write_checkpoint_hdf5(float_sw4 a_time, int a_cycle,
s << mEW->getPath() << "/";
s << fileSuffix.str();

#ifndef SW4_USE_SCR
// Keep track of the number of files, save previous file name, and delete the
// second last.
cycle_checkpoints(s.str());
#else
// SCR can delete older checkpoints, e.g., SCR_PREFIX_SIZE=3

// Inform SCR that a new checkpoint is starting
std::string cycle_num;
cycle_num << "cycle=" << a_cycle;
SCR_Start_output(cycle_num.str().c_str(), SCR_FLAG_CHECKPOINT);

// Ask SCR for the path to write our checkpoint file
// This could be moved just before each H5FCreate call if need to preserve original file name
char scr_file[SCR_MAX_FILENAME];
SCR_Route_file(s.str().c_str(), scr_file);
s = scr_file;
#endif

hid_t fid, fapl, dxpl, dspace, mspace, mydspace, dtype, dcpl;
int myrank, nrank;
Expand Down Expand Up @@ -1343,6 +1429,12 @@ void CheckPoint::write_checkpoint_hdf5(float_sw4 a_time, int a_cycle,
#else
H5Fclose(fid);
#endif

#ifdef SW4_USE_SCR
int valid = 1;
SCR_Complete_output(valid);
#endif

etime = MPI_Wtime();

if (myrank == 0)
Expand All @@ -1365,7 +1457,22 @@ void CheckPoint::read_checkpoint_hdf5(float_sw4& a_time, int& a_cycle,
s << mRestartPath << "/";
else if (mEW->getPath() != "./")
s << mEW->getPath();

#ifndef SW4_USE_SCR
s << mRestartFile;
#else
// TODO: need to get cycle_num from earlier call to Have_restart

// TODO: this is not right, but you get the idea...
std::stringstream fileSuffix;
compute_file_suffix(cycle_num, fileSuffix);
s << fileSuffix.str();

// Ask SCR for the path to open our checkpoint file
char scr_file[SCR_MAX_FILENAME];
SCR_Route_file(s.str().c_str(), scr_file);
s = scr_file;
#endif

if (myrank == 0)
std::cout << "Reading checkpoint from file " << s.str() << std::endl;
Expand Down Expand Up @@ -1474,6 +1581,11 @@ void CheckPoint::read_checkpoint_hdf5(float_sw4& a_time, int& a_cycle,
H5Pclose(fapl);
H5Pclose(dxpl);
H5Fclose(fid);

#ifdef SW4_USE_SCR
int valid = 1;
SCR_Complete_restart(valid);
#endif
}
#endif // End USE_HDF5
//-----------------------------------------------------------------------
Expand Down
1 change: 1 addition & 0 deletions src/CheckPoint.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ void write_checkpoint_scr(float_sw4 a_time, int a_cycle,
void define_pio();
void setSteps(int a_steps);
void compute_file_suffix(int cycle, std::stringstream& fileSuffix);
void compute_file_suffix(const char* cycle, std::stringstream& fileSuffix);
void cycle_checkpoints(string fname);
void write_header(int& fid, float_sw4 a_time, int a_cycle, int& hsize);
void read_header(int& fid, float_sw4& a_time, int& a_cycle, int& hsize);
Expand Down