Change tests running policy

Do not use `ctest` anymore, tests are now launch directly (manually)
using generated unit tests binaries (unit_tests and mpi_unit_tests)

`ctest` can always be invoked but not using `make test`.

The reason for this change is that `ctest` runs a binary for each test
which is not what we want now. The goal of this development is to
provide a mesh database that is built **once** for all: the cost of
building meshes could be prohibitive and their multiplication should
be avoided for obvious performances and storage reasons.

.gitlab-ci/clang10-mpi-debug.yml

@@ -21,7 +21,7 @@ test:clang10-mpi-debug:
     - mkdir -p build/clang10-debug-mpi
     - cd build/clang10-debug-mpi
     - CXX=clang++-10 CC=clang-10 cmake ../.. -DCMAKE_BUILD_TYPE=Debug
-    - make run_unit_tests
+    - make test
   cache:
     key: "${CI_COMMIT_REF_SLUG}-clang10-debug-mpi"
     paths:

.gitlab-ci/clang10-mpi-release.yml

@@ -21,7 +21,7 @@ test:clang10-mpi-release:
     - mkdir -p build/clang10-release-mpi
     - cd build/clang10-release-mpi
     - CXX=clang++-10 CC=clang-10 cmake ../.. -DCMAKE_BUILD_TYPE=Release -DCLANG_FORMAT=/usr/bin/clang-format-10
-    - make run_unit_tests
+    - make test
   cache:
     key: "${CI_COMMIT_REF_SLUG}-clang10-release-mpi"
     paths:

.gitlab-ci/gcc10-mpi-debug.yml

@@ -21,7 +21,7 @@ test:gcc10-mpi-debug:
     - mkdir -p build/gcc10-debug-mpi
     - cd build/gcc10-debug-mpi
     - CXX=g++-10 CC=gcc-10 cmake ../.. -DCMAKE_BUILD_TYPE=Debug
-    - make run_unit_tests
+    - make test
   cache:
     key: "${CI_COMMIT_REF_SLUG}-gcc10-debug-mpi"
     paths:

.gitlab-ci/gcc10-mpi-release.yml

@@ -21,7 +21,7 @@ test:gcc10-mpi-release:
     - mkdir -p build/gcc10-release-mpi
     - cd build/gcc10-release-mpi
     - CXX=g++-10 CC=gcc-10 cmake ../.. -DCMAKE_BUILD_TYPE=Release
-    - make run_unit_tests
+    - make test
   cache:
     key: "${CI_COMMIT_REF_SLUG}-gcc10-release-mpi"
     paths:

.gitlab-ci/gcc10-seq-release.yml

@@ -21,7 +21,7 @@ test:gcc10-seq-release:
     - mkdir -p build/gcc10-release-seq
     - cd build/gcc10-release-seq
     - CXX=g++-10 CC=gcc-10 cmake ../.. -DCMAKE_BUILD_TYPE=Release
-    - make run_unit_tests
+    - make test
   cache:
     key: "${CI_COMMIT_REF_SLUG}-gcc10-release-seq"
     paths:

CMakeLists.txt

@@ -314,12 +314,46 @@ include("${CATCH_MODULE_PATH}/contrib/ParseAndAddCatchTests.cmake")
 add_subdirectory("${CATCH_MODULE_PATH}")
 
 add_subdirectory(tests)
-enable_testing()
 
-add_custom_target(run_unit_tests
-  COMMAND ${CMAKE_CTEST_COMMAND}
+if(${PUGS_HAS_MPI})
+  set(MPIEXEC_OPTION_FLAGS --oversubscribe)
+  if (NOT "$ENV{GITLAB_CI}" STREQUAL "")
+    set(MPIEXEC_OPTION_FLAGS ${MPIEXEC_OPTION_FLAGS} --allow-run-as-root)
+  endif()
+  set(MPIEXEC_NUMPROC 3)
+  set(MPIEXEC_PATH_FLAG --path)
+  set(MPI_LAUNCHER ${MPIEXEC} ${MPIEXEC_NUMPROC_FLAG} ${MPIEXEC_NUMPROC} ${MPIEXEC_PATH_FLAG} ${PUGS_BINARY_DIR} ${MPIEXEC_OPTION_FLAGS})
+endif()
+
+add_custom_target(all_unit_tests
   DEPENDS unit_tests mpi_unit_tests
-  COMMENT "Executing unit tests."
+)
+
+add_custom_target(test
+  DEPENDS run_all_unit_tests
+  )
+
+add_custom_target(run_all_unit_tests
+  DEPENDS run_mpi_unit_tests
+  )
+
+if(PUGS_HAS_MPI)
+  set(RUN_MPI_UNIT_TESTS_COMMENT "Running mpi_unit_tests [using ${MPIEXEC_NUMPROC} procs]")
+else()
+  set(RUN_MPI_UNIT_TESTS_COMMENT "Running mpi_unit_tests [sequentially]")
+endif()
+
+add_custom_target(run_mpi_unit_tests
+  COMMAND ${MPI_LAUNCHER} "${PUGS_BINARY_DIR}/mpi_unit_tests"
+  DEPENDS run_unit_tests
+  COMMENT ${RUN_MPI_UNIT_TESTS_COMMENT}
+  )
+
+
+add_custom_target(run_unit_tests
+  COMMAND "${PUGS_BINARY_DIR}/unit_tests"
+  DEPENDS all_unit_tests
+  COMMENT "Running unit_tests"
   )
 
 # unit tests coverage
@@ -365,16 +399,42 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "Coverage")
 
   find_program(FASTCOV fastcov fastcov.py)
 
+  add_custom_target(coverage_unit_tests
+    ALL # in coverage mode we do coverage!
+
+    COMMAND "${PUGS_BINARY_DIR}/unit_tests"
+
+    COMMENT "Running unit_tests"
+    DEPENDS coverage_zero_counters
+    )
+
+  add_custom_target(coverage_mpi_unit_tests
+    ALL # in coverage mode we do coverage!
+
+    COMMAND ${MPI_LAUNCHER} "mpi_unit_tests"
+
+    COMMENT "Running mpi_unit_tests"
+    DEPENDS coverage_unit_tests
+    )
+
+  add_custom_target(coverage_run_all_unit_tests
+    ALL # in coverage mode we do coverage!
+
+    DEPENDS coverage_mpi_unit_tests
+    )
+
   if (FASTCOV AND (CMAKE_CXX_COMPILER_ID STREQUAL "GNU") AND (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER "9"))
 
-    add_custom_target(coverage
+    add_custom_target(coverage_zero_counters
       ALL # in coverage mode we do coverage!
 
       # zero all counters
       COMMAND ${FASTCOV} -q -z --gcov "${GCOV_BIN}"
+      DEPENDS all_unit_tests
+      )
 
-      # Run tests
-      COMMAND ${CMAKE_CTEST_COMMAND}
+    add_custom_target(coverage
+      ALL # in coverage mode we do coverage!
 
       COMMAND ${FASTCOV} -q --gcov "${GCOV_BIN}"
       --include "${PUGS_SOURCE_DIR}/src"
@@ -383,22 +443,25 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "Coverage")
 
       COMMAND ${LCOV} --gcov "${GCOV_BIN}" --list coverage.info
 
-      DEPENDS unit_tests mpi_unit_tests
+      DEPENDS coverage_run_all_unit_tests
       COMMENT "Running test coverage."
       WORKING_DIRECTORY "${PUGS_BINARY_DIR}"
       )
 
   else()
 
-    add_custom_target(coverage
+    add_custom_target(coverage_zero_counters
       ALL # in coverage mode we do coverage!
       # Cleanup previously generated profiling data
       COMMAND ${LCOV} -q --gcov "${GCOV_BIN}" --base-directory "${PUGS_BINARY_DIR}/src" --directory "${PUGS_BINARY_DIR}" --zerocounters
       # Initialize profiling data with zero coverage for every instrumented line of the project
       # This way the percentage of total lines covered will always be correct, even when not all source code files were loaded during the test(s)
       COMMAND ${LCOV} -q --gcov "${GCOV_BIN}" --base-directory "${PUGS_BINARY_DIR}/src" --directory "${PUGS_BINARY_DIR}" --capture --initial --output-file coverage_base.info
-      # Run tests
-      COMMAND ${CMAKE_CTEST_COMMAND}
+      DEPENDS all_unit_tests
+      )
+
+
+    add_custom_target(coverage
       # Collect data from executions
       COMMAND ${LCOV} --gcov "${GCOV_BIN}" --base-directory "${PUGS_BINARY_DIR}/src" --directory "${PUGS_BINARY_DIR}" --capture --output-file coverage_ctest.info
       # Combine base and ctest results
@@ -413,7 +476,7 @@ if("${CMAKE_BUILD_TYPE}" STREQUAL "Coverage")
 
       COMMAND ${LCOV} --gcov "${GCOV_BIN}" --list coverage.info
 
-      DEPENDS unit_tests mpi_unit_tests
+      DEPENDS coverage_run_all_unit_tests
       COMMENT "Running test coverage."
       WORKING_DIRECTORY "${PUGS_BINARY_DIR}"
       )

src/utils/pugs_config.hpp.in

@@ -10,5 +10,6 @@
 #cmakedefine SYSTEM_IS_WINDOWS
 
 #define PUGS_BUILD_TYPE "@CMAKE_BUILD_TYPE@"
+#define PUGS_BINARY_DIR "@PUGS_BINARY_DIR@"
 
 #endif   // PUGS_CONFIG_HPP

tests/mpi_test_main.cpp

@@ -4,8 +4,10 @@
 #include <Kokkos_Core.hpp>
 
 #include <utils/Messenger.hpp>
+#include <utils/pugs_config.hpp>
 
 #include <cstdlib>
+#include <filesystem>
 
 int
 main(int argc, char* argv[])
@@ -13,19 +15,50 @@ main(int argc, char* argv[])
   parallel::Messenger::create(argc, argv);
   Kokkos::initialize({4, -1, -1, true});
 
-  // Disable outputs from tested classes to the standard output
-  std::cout.setstate(std::ios::badbit);
+  const std::string output_base_name{"mpi_test_rank_"};
+
+  std::filesystem::path parallel_output(std::string{PUGS_BINARY_DIR});
+
+  std::filesystem::path gcov_prefix = [&]() -> std::filesystem::path {
+    std::string template_temp_dir = std::filesystem::temp_directory_path() / "pugs_gcov_XXXXXX";
+    return std::filesystem::path{mkdtemp(&template_temp_dir[0])};
+  }();
 
-  if (parallel::rank() != 0) {
-    setenv("GCOV_PREFIX", "/dev/null", 1);
-  }
   Catch::Session session;
   int result = session.applyCommandLine(argc, argv);
 
+  if (parallel::rank() != 0) {
+    // Disable outputs for ranks != 0
+    setenv("GCOV_PREFIX", gcov_prefix.string().c_str(), 1);
+
+    std::cout << "putting gcov infos in " << gcov_prefix.string() << '\n';
+
+    parallel_output /= output_base_name + std::to_string(parallel::rank());
+
+    Catch::ConfigData data{session.configData()};
+    data.outputFilename = parallel_output.string();
+    session.useConfigData(data);
+  }
+
   if (result == 0) {
     // Disable outputs from tested classes to the standard output
     std::cout.setstate(std::ios::badbit);
     result = session.run();
+
+    if (parallel::rank() == 0) {
+      if (parallel::size() > 1) {
+        session.config().stream() << rang::fgB::green << "Other rank outputs are stored in corresponding files"
+                                  << rang::style::reset << '\n';
+
+        for (size_t i_rank = 1; i_rank < parallel::size(); ++i_rank) {
+          std::filesystem::path parallel_output(std::string{PUGS_BINARY_DIR});
+          parallel_output /= output_base_name + std::to_string(i_rank);
+          session.config().stream() << " - " << rang::fg::green << parallel_output.parent_path().string()
+                                    << parallel_output.preferred_separator << rang::style::reset << rang::fgB::green
+                                    << parallel_output.filename().string() << rang::style::reset << '\n';
+        }
+      }
+    }
   }
 
   Kokkos::finalize();