From c06f7274499ad925809c151ff26d02205bcc9898 Mon Sep 17 00:00:00 2001 From: ScSteffen Date: Tue, 17 Sep 2024 13:05:53 -0400 Subject: [PATCH] fixed mpi import, fixed csv output --- CMakeLists.txt | 2 +- src/common/config.cpp | 6 +++--- src/common/io.cpp | 18 +++++++++--------- src/common/mesh.cpp | 11 +++++------ src/main.cpp | 10 +++++----- src/solvers/snsolver_hpc.cpp | 22 +++++++++++----------- 6 files changed, 34 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1680de14..58470146 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -9,7 +9,6 @@ option( BUILD_UNITY "enables unity build for faster compile times" ON ) option( BUILD_CODE_COV "enables compiler option required for code coverage analysis" OFF ) option( BUILD_ML "enables build with tensorflow backend access" OFF ) option( BUILD_MPI "enables build with MPI access" OFF ) -add_definitions(-DBUILD_MPI) ################################################# @@ -33,6 +32,7 @@ find_package( OpenMP REQUIRED ) message(STATUS "MPI build flag: ${BUILD_MPI}") if( BUILD_MPI ) + add_definitions(-DIMPORT_MPI) find_package( MPI REQUIRED ) include_directories( ${MPI_INCLUDE_PATH} ) message(STATUS "C++ compiler: ${CMAKE_CXX_COMPILER}") diff --git a/src/common/config.cpp b/src/common/config.cpp index 1483ac77..60ae732a 100644 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -5,7 +5,7 @@ * * Disclaimer: This class structure was copied and modifed with open source permission from SU2 v7.0.3 https://su2code.github.io/ */ -#ifdef BUILD_MPI +#ifdef IMPORT_MPI #include #endif #include "common/config.hpp" @@ -1207,7 +1207,7 @@ bool Config::TokenizeString( string& str, string& option_name, vector& o void Config::InitLogger() { int rank = 0; -#ifdef BUILD_MPI +#ifdef IMPORT_MPI MPI_Comm_rank( MPI_COMM_WORLD, &rank ); // Initialize MPI #endif @@ -1366,7 +1366,7 @@ void Config::InitLogger() { spdlog::flush_every( std::chrono::seconds( 5 ) ); } } -#ifdef BUILD_MPI +#ifdef IMPORT_MPI MPI_Barrier( MPI_COMM_WORLD ); #endif } diff --git a/src/common/io.cpp b/src/common/io.cpp index 47cb089d..38804953 100644 --- a/src/common/io.cpp +++ b/src/common/io.cpp @@ -15,7 +15,7 @@ #include #include -#ifdef BUILD_MPI +#ifdef IMPORT_MPI #include #endif @@ -49,14 +49,14 @@ void ExportVTK( const std::string fileName, const std::vector>>& outputFields, const std::vector>& outputFieldNames, const Mesh* mesh ) { - int rank = 0; + int rank = 0; int nprocs = 1; -#ifdef BUILD_MPI +#ifdef IMPORT_MPI // Initialize MPI MPI_Comm_size( MPI_COMM_WORLD, &nprocs ); MPI_Comm_rank( MPI_COMM_WORLD, &rank ); -#endif -if( rank == 0 ) { +#endif + if( rank == 0 ) { unsigned dim = mesh->GetDim(); unsigned numCells = mesh->GetNumCells(); unsigned numNodes = mesh->GetNumNodes(); @@ -145,8 +145,8 @@ if( rank == 0 ) { // auto log = spdlog::get( "event" ); // log->info( "Result successfully exported to '{0}'!", fileNameWithExt ); } -#ifdef BUILD_MPI - MPI_Barrier( MPI_COMM_WORLD ); +#ifdef IMPORT_MPI + MPI_Barrier( MPI_COMM_WORLD ); #endif } @@ -472,7 +472,7 @@ std::string ParseArguments( int argc, char* argv[] ) { void PrintLogHeader( std::string inputFile ) { int nprocs = 1; int rank = 0; -#ifdef BUILD_MPI +#ifdef IMPORT_MPI // Initialize MPI MPI_Comm_size( MPI_COMM_WORLD, &nprocs ); MPI_Comm_rank( MPI_COMM_WORLD, &rank ); @@ -512,7 +512,7 @@ void PrintLogHeader( std::string inputFile ) { } // log->info( "------------------------------------------------------------------------" ); } -#ifdef BUILD_MPI +#ifdef IMPORT_MPI MPI_Barrier( MPI_COMM_WORLD ); #endif } diff --git a/src/common/mesh.cpp b/src/common/mesh.cpp index 4740685a..5fd6ce6e 100644 --- a/src/common/mesh.cpp +++ b/src/common/mesh.cpp @@ -6,7 +6,7 @@ #include #include #include -#ifdef BUILD_MPI +#ifdef IMPORT_MPI #include #endif #include @@ -28,7 +28,7 @@ Mesh::Mesh( const Config* settings, } int nprocs = 1; int rank = 0; -#ifdef BUILD_MPI +#ifdef IMPORT_MPI MPI_Comm_size( MPI_COMM_WORLD, &nprocs ); MPI_Comm_rank( MPI_COMM_WORLD, &rank ); #endif @@ -95,7 +95,7 @@ Mesh::~Mesh() {} void Mesh::ComputeConnectivity() { int nprocs = 1; int rank = 0; -#ifdef BUILD_MPI +#ifdef IMPORT_MPI MPI_Comm_size( MPI_COMM_WORLD, &nprocs ); MPI_Comm_rank( MPI_COMM_WORLD, &rank ); #endif @@ -577,16 +577,15 @@ double Mesh::GetDistanceToOrigin( unsigned idx_cell ) const { return sqrt( distance ); } - unsigned Mesh::GetCellOfKoordinate( const double x, const double y ) const { // Experimental parallel implementation unsigned koordinate_cell_id = std::numeric_limits::max(); bool found = false; -//#pragma omp parallel for shared( found ) + // #pragma omp parallel for shared( found ) for( unsigned idx_cell = 0; idx_cell < _numCells; idx_cell++ ) { if( IsPointInsideCell( idx_cell, x, y ) ) { - //#pragma omp critical + // #pragma omp critical { if( !found ) { koordinate_cell_id = idx_cell; diff --git a/src/main.cpp b/src/main.cpp index 98b8980d..6782e9b8 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -4,7 +4,7 @@ * @version: 0.1 */ -#ifdef BUILD_MPI +#ifdef IMPORT_MPI #include #endif #include @@ -30,12 +30,12 @@ int main( int argc, char** argv ) { #else // wchar_t* program = Py_DecodeLocale( argv[0], NULL ); // Py_SetProgramName( program ); -#ifdef BUILD_MPI +#ifdef IMPORT_MPI MPI_Init( &argc, &argv ); printf( "| KiT-RT compiled with MPI and OpenMP parallelization\n" ); #endif -#ifndef BUILD_MPI - printf( "| KiT-RT compiled with OpenMP, but without MPI parallelization\n" ); +#ifndef IMPORT_MPI + printf( "| KiT-RT compiled with OpenMP, but without MPI parallelization\n" ); #endif std::string filename = ParseArguments( argc, argv ); @@ -70,7 +70,7 @@ int main( int argc, char** argv ) { } delete config; -#ifdef BUILD_MPI +#ifdef IMPORT_MPI MPI_Finalize(); #endif diff --git a/src/solvers/snsolver_hpc.cpp b/src/solvers/snsolver_hpc.cpp index eca7f257..8af4b407 100644 --- a/src/solvers/snsolver_hpc.cpp +++ b/src/solvers/snsolver_hpc.cpp @@ -1,4 +1,4 @@ -#ifdef BUILD_MPI +#ifdef IMPORT_MPI #include #endif #include "common/config.hpp" @@ -11,12 +11,12 @@ #include "toolboxes/textprocessingtoolbox.hpp" SNSolverHPC::SNSolverHPC( Config* settings ) { -#ifdef BUILD_MPI +#ifdef IMPORT_MPI // Initialize MPI MPI_Comm_size( MPI_COMM_WORLD, &_numProcs ); MPI_Comm_rank( MPI_COMM_WORLD, &_rank ); #endif -#ifndef BUILD_MPI +#ifndef IMPORT_MPI _numProcs = 1; // default values _rank = 0; #endif @@ -148,7 +148,7 @@ SNSolverHPC::SNSolverHPC( Config* settings ) { } // _mass += _scalarFlux[idx_cell] * _areas[idx_cell]; } -#ifdef BUILD_MPI +#ifdef IMPORT_MPI MPI_Barrier( MPI_COMM_WORLD ); #endif SetGhostCells(); @@ -156,7 +156,7 @@ SNSolverHPC::SNSolverHPC( Config* settings ) { PrepareScreenOutput(); // Screen Output PrepareHistoryOutput(); // History Output } -#ifdef BUILD_MPI +#ifdef IMPORT_MPI MPI_Barrier( MPI_COMM_WORLD ); #endif delete quad; @@ -530,12 +530,12 @@ void SNSolverHPC::FVMUpdate() { temp_scalarFlux[idx_cell] = localScalarFlux; // set flux } // MPI Allreduce: _scalarFlux -#ifdef BUILD_MPI +#ifdef IMPORT_MPI MPI_Barrier( MPI_COMM_WORLD ); MPI_Allreduce( temp_scalarFlux.data(), _scalarFlux.data(), _nCells, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD ); MPI_Barrier( MPI_COMM_WORLD ); #endif -#ifndef BUILD_MPI +#ifndef IMPORT_MPI _scalarFlux = temp_scalarFlux; #endif } @@ -679,7 +679,7 @@ void SNSolverHPC::IterPostprocessing() { } } // MPI Allreduce -#ifdef BUILD_MPI +#ifdef IMPORT_MPI double tmp_curScalarOutflow = 0.0; double tmp_curScalarOutflowPeri1 = 0.0; double tmp_curScalarOutflowPeri2 = 0.0; @@ -755,12 +755,12 @@ void SNSolverHPC::IterPostprocessing() { // probe values green ComputeQOIsGreenProbingLine(); -#ifdef BUILD_MPI +#ifdef IMPORT_MPI MPI_Barrier( MPI_COMM_WORLD ); MPI_Allreduce( temp_probingMoments.data(), _probingMoments.data(), 3 * n_probes, MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD ); MPI_Barrier( MPI_COMM_WORLD ); #endif -#ifndef BUILD_MPI +#ifndef IMPORT_MPI for( unsigned idx_probe = 0; idx_probe < n_probes; idx_probe++ ) { // Loop over probing cells _probingMoments[Idx2D( idx_probe, 0, 3 )] = temp_probingMoments[Idx2D( idx_probe, 0, 3 )]; _probingMoments[Idx2D( idx_probe, 1, 3 )] = temp_probingMoments[Idx2D( idx_probe, 1, 3 )]; @@ -1147,7 +1147,7 @@ void SNSolverHPC::PrintHistoryOutput( unsigned idx_iter ) { } lineToPrint += tmp + ","; } - tmp = TextProcessingToolbox::DoubleToScientificNotation( _historyOutputFields[_settings->GetNScreenOutput() - 1] ); + tmp = TextProcessingToolbox::DoubleToScientificNotation( _historyOutputFields[_settings->GetNHistoryOutput() - 1] ); lineToPrint += tmp; // Last element without comma if( _settings->GetHistoryOutputFrequency() != 0 && idx_iter % (unsigned)_settings->GetHistoryOutputFrequency() == 0 ) {