Merge pull request #248 from ngs333/cpp_dev

Updating GFDL repo with numerous commits.

cpp/CMakeLists.txt

@@ -12,12 +12,15 @@ set(CMAKE_CXX_STANDARD 20)
 add_subdirectory(dependencies)
 
 #Testing:
-#NOTE: The inital framework are Catch2 + ApprovalTest. This should provide easy
+#NOTE: The initial framework are Catch2 + ApprovalTest. This should provide easy
 # unit testing, acceptance testing, and micro benchmarking. However, it may be
 # convenient to directly use and add cmakes Catch via "include(Catch)". E.g.
 # can nccmp be directly called?"
-enable_testing()
-add_subdirectory(test)
+if(BUILD_TESTS)
+    enable_testing()
+    add_subdirectory(test)
+endif(BUILD_TESTS)
+
 
 
 # The " {CMAKE_SOURCE_DIR}/include" directories below is only temporary as
@@ -32,9 +35,21 @@ include_directories(${CMAKE_SOURCE_DIR}/include)
 ##if (NOT CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
 ##endif()
 
+##with-mpi option not yet available in the C++ codebase
+##option(with-mpi "Use legacy MPI and legacy create_xgrid in fregrid build" OFF)
+option(with-gpu "Compile for GPU. Only use with nvc++" OFF )
+## These experimental options are only being used by developers:
+##TODO: remove SYCL use from the C++ codebase
+option(with-sycl "Compile with  SYCL *** WARNING: EXPERIMENTAL *** " OFF)
 #find_package(IntelSYCL REQUIRED)
-option(USE_SYCL "Use SYCL compiling search and other components" OFF)
-option(USE_NEXTAFTER "Use nextafter expanded bounding intervals" OFF)
+
+if(with-gpu)
+    add_definitions(-DWITH_GPU)
+endif()
+
+if(with-sycl)
+    add_definitions(-DWITH_SYCL)
+endif()
 
 ##Location of some libs is external to this project.
 #include_directories(/home/mzuniga/NetCdCPP/install/include)
@@ -51,11 +66,10 @@ add_subdirectory(common)
 add_subdirectory(search)
 add_subdirectory(libfrencutils)
 add_subdirectory(fregrid)
-if(USE_SYCL)
-    add_subdirectory(sycl_tests)
-endif(USE_SYCL)
 
-#add_subdirectory(vendor/edlib EXCLUDE_FROM_ALL)
+if(with-sycl)
+    add_subdirectory(sycl_tests)
+endif(with-sycl)
 
 add_dependencies(searchtest.x libSearch)
 add_dependencies(fregrid.x libSearch libFrencu)

cpp/CMakePresets.json

@@ -0,0 +1,66 @@
+{
+  "version": 6,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 21,
+    "patch": 0
+  },
+  "configurePresets": [
+    {
+      "name": "gcc-debug-pr",
+      "displayName": "GCC-DEBUG",
+      "description": "Debug build options for GCC",
+      "generator": "Unix Makefiles",
+      "binaryDir": "${sourceDir}/build_gcc_debug_pr",
+      "toolchainFile":"${sourceDir}/gcc_toolchain",
+      "cacheVariables": {
+        "CMAKE_CXX_FLAGS_INIT": "-g -O0 -Wall  -Wpedantic -Wextra"
+      }
+    },
+    {
+      "name": "gcc-release-pr",
+      "displayName": "GCC-RELEASE",
+      "description": "Release build options for GCC",
+      "generator": "Unix Makefiles",
+      "binaryDir": "${sourceDir}/build_gcc_release_pr",
+      "toolchainFile":"${sourceDir}/gcc_toolchain",
+      "cacheVariables": {
+        "CMAKE_CXX_FLAGS_INT": "-g -O2 -Wall  -Wpedantic -Wextra"
+      }
+    },
+    {
+      "name": "gpu-nvhpc-pr",
+      "displayName": "GPU-NVHPC",
+      "description": "NVC++ build option for gpu with std::par",
+      "generator": "Unix Makefiles",
+      "binaryDir": "${sourceDir}/build_gpu_nvhpc_pr",
+      "toolchainFile":"${sourceDir}/nvhpc_toolchain",
+      "cacheVariables": {
+        "CMAKE_CXX_FLAGS_INIT": "-O2 -std=c++20 -stdpar=gpu --gcc-toolchain=/opt/gcc/12.3.0 -I/opt/gcc/12.3.0/include/c++/12.3.0 -I/opt/netcdf/4.9.2/NVHPC/23.7/include -I/opt/hdf5/1.14.1-2/NVHPC/23.7/include"
+      }
+    }
+  ],
+  "buildPresets": [
+    {
+      "name": "gcc-debug-build-pr",
+      "displayName": "GCC-DEBUG-BUILD",
+      "targets": [ "fregrid.x"],
+      "verbose": true,
+      "configurePreset": "gcc-debug-pr"
+    },
+    {
+      "name": "gcc-release-build-pr",
+      "displayName": "GCC-RELEASE-BUILD",
+      "targets": ["fregrid.x"],
+      "verbose": true,
+      "configurePreset": "gcc-release-pr"
+    },
+    {
+      "name": "gpu-nvhpc-build-pr",
+      "displayName": "GPU-NVHPC-BUILD",
+      "targets": ["fregrid.x"],
+      "verbose": true,
+      "configurePreset": "gpu-nvhpc-pr"
+    }
+  ]
+}

cpp/README_BUILD.txt

@@ -0,0 +1,24 @@
+#   I) load desired compilers and libs
+
+#   IIA) #Compiling with CMake presets:
+ # inspect the preset files under dir  FRE-NCtools/cpp
+ # if you dont find one to you liking, modify one or make a
+ # new one
+ # List the available presets:
+ cd FRE-NCtools/cpp
+ cmake --list-presets
+ #choose one of the presets, e.g. gpu-nvhpc-pr :
+ cmake . --preset=gpu-nvhpc-pr -Dwith-gpu=ON
+ #cd to the directory listed in the last output line - its
+ #  where the build files are written.
+ cd build_gpu_nvhpc_pr
+ # run cmake in verbose mode
+ cmake --build . -v
+ # note the executables are currently copied to dir FRE-NCtools
+-----------------
+ #  IIB) Compiling using env vars and more cmake command line options.
+ #set the compiler
+ export CXX=/opt/nvidia/hpc_sdk/Linux_x86_64/23.7/compilers/bin/nvc++
+ #pass to cmake the flags you want.
+ cmake .. -Dwith-gpu="ON" -DCMAKE_CXX_FLAGS="-O2 -std=c++20 -stdpar=gpu"
+ cmake  --build . -v

cpp/fregrid/CMakeLists.txt

@@ -24,6 +24,7 @@ add_executable(fregrid.x fregrid.C conserve_interp.C fregrid_util.C bilinear_int
 add_executable(searchtest.x searchtest.C)
 #add_sycl_to_target(TARGET searchtest.x SOURCES  searchtest.C) #in the future, may require other .C files
 
+#add tbb ?
 target_link_libraries(searchtest.x libSearch tbb)
 target_link_libraries(fregrid.x  libFrencu libSearch netcdf tbb)
 

cpp/fregrid/conserve_interp.C

@@ -48,10 +48,12 @@ void setup_conserve_interp(int ntiles_in, const Grid_config *grid_in, int ntiles
                            Grid_config *grid_out, Interp_config *interp, unsigned int opcode) {
   int n, m, ii, nx_in, ny_in, nx_out, ny_out, tile;
   size_t nxgrid, nxgrid_prev;
-  int *i_in = NULL, *j_in = NULL, *i_out = NULL, *j_out = NULL;
   int *tmp_t_in = NULL, *tmp_i_in = NULL, *tmp_j_in = NULL, *tmp_i_out = NULL, *tmp_j_out = NULL;
   double *tmp_di_in, *tmp_dj_in;
-  double *xgrid_area = NULL, *tmp_area = NULL, *xgrid_clon = NULL, *xgrid_clat = NULL;
+
+  int *i_in{nullptr}, *j_in{nullptr}, *i_out{nullptr}, *j_out{nullptr};
+  double *xgrid_area{nullptr}, *xgrid_clon{nullptr}, *xgrid_clat{nullptr}, *tmp_area{nullptr};
+
 
   double garea;
   typedef struct {
@@ -129,14 +131,14 @@ void setup_conserve_interp(int ntiles_in, const Grid_config *grid_in, int ntiles
     }
     if (mpp_pe() == mpp_root_pe())
       printf("NOTE: Finish reading index and weight for conservative interpolation from file.\n");
-  } else {
-    i_in = (int *) malloc(MAXXGRID * sizeof(int));
-    j_in = (int *) malloc(MAXXGRID * sizeof(int));
-    i_out = (int *) malloc(MAXXGRID * sizeof(int));
-    j_out = (int *) malloc(MAXXGRID * sizeof(int));
-    xgrid_area = (double *) malloc(MAXXGRID * sizeof(double));
-    xgrid_clon = (double *) malloc(MAXXGRID * sizeof(double));
-    xgrid_clat = (double *) malloc(MAXXGRID * sizeof(double));;
+  } else { //Calculate the xgrid instead of reading it in.
+    //Note: The newer create_xgrid_2dx2d_... algorithms (e.g. create_xgrid_2dx2d_order2
+    // with search tree or GPU) allocates only the needed memory. So routines called here
+    // were modified to allocate their own memory. Eventaully, all of them should be modified
+    // to use std::vector (vs malloc arrays) for fields like xgrid_area, i_in, and j_out, etc.
+    // This may be particularly easy if MPI version of the code can be removed altogether
+    // for fields like xgrid_area, i_in, j_out etc.
+
     cell_in = (CellStruct *) malloc(ntiles_in * sizeof(CellStruct));
     for (m = 0; m < ntiles_in; m++) {
       nx_in = grid_in[m].nx;
@@ -334,7 +336,7 @@ void setup_conserve_interp(int ntiles_in, const Grid_config *grid_in, int ntiles
         nx_in = grid_in[n].nx;
         ny_in = grid_in[n].ny;
         for (j = 0; j < ny_in; j++)
-          for (size_t i = 0; i < (size_t)nx_in; i++) {
+          for (size_t i = 0; i < (size_t) nx_in; i++) {
             ii = j * nx_in + i;
             if (cell_in[n].area[ii] > 0) {
               if (fabs(cell_in[n].area[ii] - grid_in[n].cell_area[ii]) / grid_in[n].cell_area[ii] <
@@ -488,8 +490,8 @@ void setup_conserve_interp(int ntiles_in, const Grid_config *grid_in, int ntiles
       max_i = 0;
       max_j = 0;
       /* comparing area1 and area2 */
-      for (unsigned int  j = 0; j < static_cast<unsigned int>(ny1); j++)
-        for (unsigned int  i = 0; i < static_cast<unsigned int>(nx1); i++) {
+      for (unsigned int j = 0; j < static_cast<unsigned int>(ny1); j++)
+        for (unsigned int i = 0; i < static_cast<unsigned int>(nx1); i++) {
           ii = j * nx1 + i;
           ratio_change = fabs(grid_out[n].cell_area[ii] - area2[ii]) / grid_out[n].cell_area[ii];
           if (ratio_change > max_ratio) {
@@ -507,9 +509,7 @@ void setup_conserve_interp(int ntiles_in, const Grid_config *grid_in, int ntiles
              grid_out[n].cell_area[ii], area2[ii]);
 
     }
-
     free(area2);
-
   }
 
   free(i_in);

cpp/gcc_toolchain

@@ -0,0 +1,3 @@
+set(tools /usr)
+set(CMAKE_C_COMPILER ${tools}/bin/gcc)
+set(CMAKE_CXX_COMPILER ${tools}/bin/g++)