Skip to content

Some multi-threading fixes #9302

New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

Merged
merged 2 commits into from
Aug 24, 2021
Merged

Some multi-threading fixes #9302

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
12192f1
ompi/request: Add a read memory barrier to sync the receive buffer so…
AboorvaDevarajan May 29, 2020
babfd93
Allow mpi_init_thread to override the MPI_THREAD_LEVEL
AboorvaDevarajan Jan 5, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion ompi/mpi/c/init_thread.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ int MPI_Init_thread(int *argc, char ***argv, int required,
int *provided)
{
int err, safe_required = MPI_THREAD_SERIALIZED;
char *env;

ompi_hook_base_mpi_init_thread_top(argc, argv, required, provided);

Expand All @@ -56,7 +57,13 @@ int MPI_Init_thread(int *argc, char ***argv, int required,
*/
if( (MPI_THREAD_SINGLE == required) || (MPI_THREAD_SERIALIZED == required) ||
(MPI_THREAD_FUNNELED == required) || (MPI_THREAD_MULTIPLE == required) ) {
safe_required = required;

if (NULL != (env = getenv("OMPI_MPI_THREAD_LEVEL"))) {
safe_required = atoi(env);
}
else {
safe_required = required;
}
}

*provided = safe_required;
Expand Down
55 changes: 30 additions & 25 deletions ompi/request/request.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -448,39 +448,44 @@ static inline bool ompi_request_tag_is_collective(int tag) {

static inline void ompi_request_wait_completion(ompi_request_t *req)
{
if (opal_using_threads () && !REQUEST_COMPLETE(req)) {
void *_tmp_ptr;
ompi_wait_sync_t sync;
if (opal_using_threads ()) {
if(!REQUEST_COMPLETE(req)) {
void *_tmp_ptr;
ompi_wait_sync_t sync;


#if OPAL_ENABLE_FT_MPI
redo:
if(OPAL_UNLIKELY( ompi_request_is_failed(req) )) {
return;
}
redo:
if(OPAL_UNLIKELY( ompi_request_is_failed(req) )) {
return;
}
#endif /* OPAL_ENABLE_FT_MPI */
_tmp_ptr = REQUEST_PENDING;
_tmp_ptr = REQUEST_PENDING;

WAIT_SYNC_INIT(&sync, 1);
WAIT_SYNC_INIT(&sync, 1);

if (OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&req->req_complete, &_tmp_ptr, &sync)) {
SYNC_WAIT(&sync);
} else {
/* completed before we had a chance to swap in the sync object */
WAIT_SYNC_SIGNALLED(&sync);
}
if (OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&req->req_complete, &_tmp_ptr, &sync)) {
SYNC_WAIT(&sync);
} else {
/* completed before we had a chance to swap in the sync object */
WAIT_SYNC_SIGNALLED(&sync);
}

#if OPAL_ENABLE_FT_MPI
if (OPAL_UNLIKELY(OMPI_SUCCESS != sync.status)) {
OPAL_OUTPUT_VERBOSE((50, ompi_ftmpi_output_handle, "Status %d reported for sync %p rearming req %p", sync.status, (void*)&sync, (void*)req));
_tmp_ptr = &sync;
if (OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&req->req_complete, &_tmp_ptr, REQUEST_PENDING)) {
opal_output_verbose(10, ompi_ftmpi_output_handle, "Status %d reported for sync %p rearmed req %p", sync.status, (void*)&sync, (void*)req);
WAIT_SYNC_RELEASE(&sync);
goto redo;
if (OPAL_UNLIKELY(OMPI_SUCCESS != sync.status)) {
OPAL_OUTPUT_VERBOSE((50, ompi_ftmpi_output_handle, "Status %d reported for sync %p rearming req %p", sync.status, (void*)&sync, (void*)req));
_tmp_ptr = &sync;
if (OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&req->req_complete, &_tmp_ptr, REQUEST_PENDING)) {
opal_output_verbose(10, ompi_ftmpi_output_handle, "Status %d reported for sync %p rearmed req %p", sync.status, (void*)&sync, (void*)req);
WAIT_SYNC_RELEASE(&sync);
goto redo;
}
}
}
#endif /* OPAL_ENABLE_FT_MPI */
assert(REQUEST_COMPLETE(req));
WAIT_SYNC_RELEASE(&sync);
assert(REQUEST_COMPLETE(req));
WAIT_SYNC_RELEASE(&sync);
}
opal_atomic_rmb();
} else {
while(!REQUEST_COMPLETE(req)) {
opal_progress();
Expand Down