ItemValueSynchronizer.hpp

#ifndef ITEM_VALUE_SYNCHRONIZER_HPP
#define ITEM_VALUE_SYNCHRONIZER_HPP

#include <ItemValue.hpp>
#include <Connectivity.hpp>

#include <unordered_map>

class ItemValueSynchronizer
{
  template <typename ConnectivityType,
            typename DataType,
            ItemType item_type,
            typename ConnectivityPtr>
  PASTIS_INLINE
  void _synchronize(const ConnectivityType& connectivity,
                    ItemValue<DataType, item_type, ConnectivityPtr>& item_value)
  {
    static_assert(not std::is_abstract_v<ConnectivityType>,
                  "_synchronize must be called on a concrete connectivity");
    const auto& item_owner =  connectivity.template owner<item_type>();

    using ItemId = ItemIdT<item_type>;
    std::vector<std::vector<ItemId>> ghost_items_per_proc(parallel::size());
    for (ItemId item_id=0; item_id<item_value.size(); ++item_id) {
      if (const size_t owner = item_owner[item_id]; owner != parallel::rank()) {
        ghost_items_per_proc[owner].emplace_back(item_id);
      }
    }

    Array<unsigned int> local_number_of_requested_values(parallel::size());
    for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
      local_number_of_requested_values[i_rank] = ghost_items_per_proc[i_rank].size();
    }

    Array<unsigned int> local_number_of_values_to_send
        = parallel::allToAll(local_number_of_requested_values);

    std::vector<Array<const int>> requested_item_number_list_by_proc(parallel::size());
    const auto& item_number = connectivity.template number<item_type>();
    for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
      const auto& ghost_items = ghost_items_per_proc[i_rank];
      Array<int> item_number_list(ghost_items.size());
      for (size_t i_item = 0; i_item<ghost_items.size(); ++i_item) {
        item_number_list[i_item] = item_number[ghost_items[i_item]];
      }
      requested_item_number_list_by_proc[i_rank] = item_number_list;
    }

    std::vector<Array<int>> to_send_item_number_list_by_proc(parallel::size());
    for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
      to_send_item_number_list_by_proc[i_rank] = Array<int>{local_number_of_values_to_send[i_rank]};
    }

    parallel::exchange(requested_item_number_list_by_proc, to_send_item_number_list_by_proc);

    std::unordered_map<int, ItemId> item_number_to_id_correspondance(connectivity.template numberOf<item_type>());
    for (ItemId item_id=0; item_id<item_number.size(); ++item_id) {
      item_number_to_id_correspondance[item_number[item_id]] = item_id;
    }

    std::vector<Array<const ItemId>> to_send_item_id_list_by_proc(parallel::size());
    for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
      Array<ItemId> to_send_item_id{local_number_of_values_to_send[i_rank]};
      const Array<int>& to_send_item_number = to_send_item_number_list_by_proc[i_rank];
      for (size_t i=0; i<to_send_item_number.size(); ++i) {
        to_send_item_id[i] = item_number_to_id_correspondance[to_send_item_number[i]];
      }
      to_send_item_id_list_by_proc[i_rank] = to_send_item_id;
    }

    std::vector<Array<const DataType>> to_send_data_by_proc(parallel::size());
    for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
      Array<DataType> to_send_data{local_number_of_values_to_send[i_rank]};
      const Array<const ItemId>& to_send_item_id = to_send_item_id_list_by_proc[i_rank];
      for (size_t i=0; i<to_send_item_id.size(); ++i) {
        to_send_data[i] = item_value[to_send_item_id[i]];
      }
      to_send_data_by_proc[i_rank] = to_send_data;
    }

    std::vector<Array<DataType>> requested_data_list_by_proc(parallel::size());
    for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
      const auto& ghost_items = ghost_items_per_proc[i_rank];
      requested_data_list_by_proc[i_rank] = Array<DataType>{ghost_items.size()};
    }

    parallel::exchange(to_send_data_by_proc, requested_data_list_by_proc);

    for (size_t i_rank=0; i_rank<parallel::size(); ++i_rank) {
      const auto& ghost_items = ghost_items_per_proc[i_rank];
      const auto& requested_data = requested_data_list_by_proc[i_rank];
      for (size_t i=0; i<ghost_items.size(); ++i) {
        item_value[ghost_items[i]] = requested_data[i];
      }
      requested_data_list_by_proc[i_rank] = Array<DataType>{ghost_items.size()};
    }
  }

 public:
  template <typename DataType,
            ItemType item_type,
            typename ConnectivityPtr>
  PASTIS_INLINE
  void synchronize(ItemValue<DataType, item_type, ConnectivityPtr>& item_value)
  {
    static int cpt=0;
    pout() << "Calling synchronize(" << cpt++ << ")...\n";

    Assert(item_value.connectivity_ptr().use_count()>0, "No connectivity is associated to this ItemValue");
    const IConnectivity& connectivity = *item_value.connectivity_ptr();

    switch (connectivity.dimension()) {
      case 1: {
        this->_synchronize(static_cast<const Connectivity1D&>(connectivity), item_value);
        break;
      }
      case 2: {
        this->_synchronize(static_cast<const Connectivity2D&>(connectivity), item_value);
        break;
      }
      case 3: {
        this->_synchronize(static_cast<const Connectivity3D&>(connectivity), item_value);
        break;
      }
      default: {
        perr() << __FILE__ << ':' << __LINE__ << ": unexpected dimension\n";
        std::terminate();
      }
    }
  }

  PASTIS_INLINE
  ItemValueSynchronizer()
  {
    ;
  }
};

#endif // ITEM_VALUE_SYNCHRONIZER_HPP