-
Notifications
You must be signed in to change notification settings - Fork 981
Expand file tree
/
Copy pathmpi_structure.cpp
More file actions
205 lines (186 loc) · 7.21 KB
/
Copy pathmpi_structure.cpp
File metadata and controls
205 lines (186 loc) · 7.21 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
/*!
* \file mpi_structure.cpp
* \brief Main subroutines for the mpi structures.
* \author T. Albring
* \version 8.5.0 "Harrier"
*
* SU2 Project Website: https://su2code.github.io
*
* The SU2 Project is maintained by the SU2 Foundation
* (http://su2foundation.org)
*
* Copyright 2012-2026, SU2 Contributors (cf. AUTHORS.md)
*
* SU2 is free software; you can redistribute it and/or
* modify it under the terms of the GNU Lesser General Public
* License as published by the Free Software Foundation; either
* version 2.1 of the License, or (at your option) any later version.
*
* SU2 is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
* Lesser General Public License for more details.
*
* You should have received a copy of the GNU Lesser General Public
* License along with SU2. If not, see <http://www.gnu.org/licenses/>.
*/
#include "mpi_structure.hpp"
#include <cstring> // memcpy
/* Initialise the MPI Communicator Rank, Size, and default MPI Communicator */
#ifdef HAVE_MPI
int CBaseMPIWrapper::Rank = 0;
int CBaseMPIWrapper::Size = 1;
CBaseMPIWrapper::Comm CBaseMPIWrapper::currentComm = MPI_COMM_WORLD;
#else
template <typename ScalarType>
int CBaseMPIWrapper<ScalarType>::Rank = 0;
template <typename ScalarType>
int CBaseMPIWrapper<ScalarType>::Size = 1;
template <typename ScalarType>
typename CBaseMPIWrapper<ScalarType>::Comm CBaseMPIWrapper<ScalarType>::currentComm = 0; // dummy value
#endif
#ifdef HAVE_MPI
int CBaseMPIWrapper::MinRankError;
bool CBaseMPIWrapper::winMinRankErrorInUse = false;
CBaseMPIWrapper::Win CBaseMPIWrapper::winMinRankError;
void CBaseMPIWrapper::Error(const std::string& ErrorMsg, const std::string& FunctionName) {
/* Set MinRankError to Rank, as the error message is called on this rank. */
MinRankError = Rank;
int flag = 0;
#if MPI_VERSION >= 3
/* Find out whether the error call is collective via MPI_Ibarrier. */
Request barrierRequest;
MPI_Ibarrier(currentComm, &barrierRequest);
/* Try to complete the non-blocking barrier call for a second. */
double startTime = SU2_MPI::Wtime();
while (true) {
MPI_Test(&barrierRequest, &flag, MPI_STATUS_IGNORE);
if (flag) break;
double currentTime = SU2_MPI::Wtime();
if (currentTime > startTime + 1.0) break;
}
#else
/* MPI_Ibarrier function is not supported. Simply wait for one
second to give other ranks the opportunity to reach this point. */
#ifdef _MSC_VER
_sleep(1);
#else
sleep(1);
#endif
#endif
if (flag) {
/* The barrier is completed and hence the error call is collective.
Set MinRankError to 0. */
MinRankError = 0;
} else {
/* The error call is not collective and the minimum rank must be
determined by one sided communication. Loop over the lower numbered
ranks to check if they participate in the error message. */
for (int i = 0; i < Rank; ++i) {
int MinRankErrorOther;
MPI_Win_lock(MPI_LOCK_SHARED, i, 0, winMinRankError);
MPI_Get(&MinRankErrorOther, 1, MPI_INT, i, 0, 1, MPI_INT, winMinRankError);
MPI_Win_unlock(i, winMinRankError);
if (MinRankErrorOther < MinRankError) {
MinRankError = MinRankErrorOther;
break;
}
}
}
/* Check if this rank must write the error message and do so. */
if (Rank == MinRankError) {
std::cout << std::endl << std::endl;
std::cout << "Error in \"" << FunctionName << "\": " << std::endl;
std::cout << "-------------------------------------------------------------------------" << std::endl;
std::cout << ErrorMsg << std::endl;
std::cout << "------------------------------ Error Exit -------------------------------" << std::endl;
std::cout << std::endl << std::endl;
}
Abort(currentComm, EXIT_FAILURE);
}
void CBaseMPIWrapper::CopyData(const void* sendbuf, void* recvbuf, int size, Datatype datatype, int recvshift,
int sendshift) {
if (datatype == MPI_DOUBLE) {
for (int i = 0; i < size; i++) {
static_cast<su2double*>(recvbuf)[i + recvshift] = static_cast<const su2double*>(sendbuf)[i + sendshift];
}
} else {
int scalarsize;
MPI_Type_size(datatype, &scalarsize);
const char* src = static_cast<const char*>(sendbuf) + sendshift * static_cast<size_t>(scalarsize);
char* dest = static_cast<char*>(recvbuf) + recvshift * static_cast<size_t>(scalarsize);
std::memcpy(static_cast<void*>(dest), static_cast<const void*>(src), size * static_cast<size_t>(scalarsize));
}
}
#else // HAVE_MPI
template <typename ScalarType>
void CBaseMPIWrapper<ScalarType>::Error(const std::string& ErrorMsg, const std::string& FunctionName) {
if (Rank == 0) {
std::cout << std::endl << std::endl;
std::cout << "Error in \"" << FunctionName << "\": " << std::endl;
std::cout << "-------------------------------------------------------------------------" << std::endl;
std::cout << ErrorMsg << std::endl;
std::cout << "------------------------------ Error Exit -------------------------------" << std::endl;
std::cout << std::endl << std::endl;
}
Abort(currentComm, 0);
}
template <typename ScalarType>
void CBaseMPIWrapper<ScalarType>::CopyData(const void* sendbuf, void* recvbuf, int size, Datatype datatype,
int recvshift, int sendshift) {
switch (datatype) {
case MPI_DOUBLE:
for (int i = 0; i < size; i++) {
static_cast<ScalarType*>(recvbuf)[i + recvshift] = static_cast<const ScalarType*>(sendbuf)[i + sendshift];
}
break;
case MPI_UNSIGNED_LONG:
for (int i = 0; i < size; i++) {
static_cast<unsigned long*>(recvbuf)[i + recvshift] = static_cast<const unsigned long*>(sendbuf)[i + sendshift];
}
break;
case MPI_LONG:
for (int i = 0; i < size; i++) {
static_cast<long*>(recvbuf)[i + recvshift] = static_cast<const long*>(sendbuf)[i + sendshift];
}
break;
case MPI_UNSIGNED_SHORT:
for (int i = 0; i < size; i++) {
static_cast<unsigned short*>(recvbuf)[i + recvshift] =
static_cast<const unsigned short*>(sendbuf)[i + sendshift];
}
break;
case MPI_CHAR:
for (int i = 0; i < size; i++) {
static_cast<char*>(recvbuf)[i + recvshift] = static_cast<const char*>(sendbuf)[i + sendshift];
}
break;
case MPI_SHORT:
for (int i = 0; i < size; i++) {
static_cast<short*>(recvbuf)[i + recvshift] = static_cast<const short*>(sendbuf)[i + sendshift];
}
break;
case MPI_INT:
for (int i = 0; i < size; i++) {
static_cast<int*>(recvbuf)[i + recvshift] = static_cast<const int*>(sendbuf)[i + sendshift];
}
break;
default:
Error("Unknown type", CURRENT_FUNCTION);
break;
};
}
template class CBaseMPIWrapper<su2double>;
#if defined CODI_REVERSE_TYPE
template class CBaseMPIWrapper<passivedouble>;
#endif
#if defined(USE_MIXED_PRECISION) && !defined(USE_SINGLE_PRECISION)
template class CBaseMPIWrapper<su2mixedfloat>;
#endif
#endif // HAVE_MPI
#ifdef HAVE_MPI
#if defined CODI_REVERSE_TYPE || defined CODI_FORWARD_TYPE
MediTypes* mediTypes;
#include <medi/medi.cpp>
#endif // defined CODI_REVERSE_TYPE || defined CODI_FORWARD_TYPE
#endif // HAVE_MPI