Changeset 9f1c84

Timestamp:

Sep 13, 2025, 2:44:01 PM (6 weeks ago)

Author:

Frederik Heber <frederik.heber@…>

Branches:

Candidate_v1.7.0, stable

Children:

9259c3

Parents:

5d18f3

git-author:

Frederik Heber <frederik.heber@…> (09/01/25 20:22:38)

git-committer:

Frederik Heber <frederik.heber@…> (09/13/25 14:44:01)

Message:

VMG: OMPI 2.1.1 has issues on writing files in parallel.

• for this reason, we deactivate the writing of the grid for open boundary conditions after computing boundary values via the greens function.
• FIX: CreateSerialOutputFile() returned pointer to scope-internal MPI_File.
• modified CreateParallelOutputFile() to use stringstreams for creating the filenames.

Location:

ThirdParty/vmg/src

Files:

• comm/comm_mpi.cpp (modified) (9 diffs)
• comm/comm_mpi.hpp (modified) (1 diff)
• discretization/boundary_value_setter.cpp (modified) (1 diff)

ThirdParty/vmg/src/comm/comm_mpi.cpp

@@ -53 +53 @@
 #include <cstring>
 #include <sstream>
+#include <iomanip>
 
 #include "base/helper.hpp"
@@ -736 +737 @@
   if (comm != MPI_COMM_NULL) {
 
-    MPI_File file;
     std::string conv_information = Helper::ReplaceWhitespaces(information, "_");
 
@@ -742 +742 @@
         begin_global, end_global, begin_local, end_local);
 
-    file = CreateSerialOutputFile(grid, comm, output_count, conv_information.c_str(),
+    MPI_File *file = CreateSerialOutputFile(grid, comm, output_count, conv_information.c_str(),
         begin_global, end_global, begin_local, end_local);
 
     char *char_buf = Helper::GetCharArray(serial_data.str());
-    MPI_File_write(file, char_buf, serial_data.str().size(), MPI_CHAR, MPI_STATUS_IGNORE);
+    MPI_File_write(*file, char_buf, serial_data.str().size(), MPI_CHAR, MPI_STATUS_IGNORE);
     delete [] char_buf;
 
-    FinalizeSerialOutputFile(file);
+    FinalizeSerialOutputFile(*file);
+
+    free(file);
 
   }
@@ -761 +763 @@
   int rank;
   MPI_File file;
-  char parallel_filename[513], serial_filename[513];
   std::stringstream buf;
-
-  MPI_Comm_rank(comm, &rank);
-
-  sprintf(parallel_filename, "%s%04d.pvti", OutputPath().c_str(), output_count);
-  sprintf(serial_filename, "%04d_%d.vti", output_count, rank);
-
-  MPI_File_open(comm, parallel_filename, MPI_MODE_WRONLY|MPI_MODE_CREATE, MPI_INFO_NULL, &file);
+  std::stringstream parallel_filename, serial_filename;
+
+  MPI_Comm_rank(comm_global, &rank);
+
+  parallel_filename << OutputPath() << std::setfill('0') << std::setw(4) << output_count << std::setw(0) <<  ".pvti";
+  serial_filename << std::setfill('0') << std::setw(4) <<  output_count << "_" << rank << std::setw(0) <<  ".pvti";
+  // sprintf(parallel_filename, "%s%04d.pvti", OutputPath().c_str(), output_count);
+  // sprintf(serial_filename, "%04d_%d.vti", output_count, rank);
+
+  MPI_File_open(comm_global, parallel_filename.str().c_str(), MPI_MODE_WRONLY|MPI_MODE_CREATE|MPI_MODE_SEQUENTIAL, MPI_INFO_NULL, &file);
   MPI_File_set_size(file, 0);
 
@@ -807 +811 @@
       buf << begin_local[i] << " " << end_local[i]  << " ";
 
-    buf << "\" Source=\"" << serial_filename << "\"/>" << std::endl;
+    buf << "\" Source=\"" << serial_filename.str() << "\"/>" << std::endl;
   }
 
@@ -833 +837 @@
 }
 
-MPI_File CommMPI::CreateSerialOutputFile(const Grid& grid, MPI_Comm& comm,
+MPI_File* CommMPI::CreateSerialOutputFile(const Grid& grid, MPI_Comm& comm,
     const int& output_count, const char* information,
     const Index& begin_global, const Index& end_global,
@@ -840 +844 @@
   char serial_filename[513];
   int rank;
-  MPI_File file;
+  MPI_File *file = (MPI_File *)malloc(sizeof(MPI_File));
   std::stringstream buf;
 
@@ -847 +851 @@
   sprintf(serial_filename, "%s%04d_%d.vti", OutputPath().c_str(), output_count, rank);
 
-  MPI_File_open(MPI_COMM_SELF, serial_filename, MPI_MODE_WRONLY|MPI_MODE_CREATE, MPI_INFO_NULL, &file);
+  MPI_File_open(MPI_COMM_SELF, serial_filename, MPI_MODE_WRONLY|MPI_MODE_CREATE, MPI_INFO_NULL, file);
 
   buf << "<?xml version=\"1.0\"?>" << std::endl
@@ -874 +878 @@
 
   char* char_buf = Helper::GetCharArray(buf.str());
-  MPI_File_write(file, char_buf, buf.str().size(), MPI_CHAR, MPI_STATUS_IGNORE);
+  MPI_File_write(*file, char_buf, buf.str().size(), MPI_CHAR, MPI_STATUS_IGNORE);
   delete [] char_buf;
 

ThirdParty/vmg/src/comm/comm_mpi.hpp

@@ -159 +159 @@
                                 const Index& begin_local, const Index& end_local);
 
-  MPI_File CreateSerialOutputFile(const Grid& grid, MPI_Comm& comm,
+  MPI_File* CreateSerialOutputFile(const Grid& grid, MPI_Comm& comm,
                                   const int& output_count, const char* information,
                                   const Index& begin_global, const Index& end_global,

ThirdParty/vmg/src/discretization/boundary_value_setter.cpp

@@ -62 +62 @@
   }
 
-  MG::GetComm()->PrintGrid(rhs_gml, "DEBUG GREEN BOUNDARY");
+  // MG::GetComm()->PrintGrid(rhs_gml, "DEBUG GREEN BOUNDARY");
 }