ConnectivityDispatcher.cpp

#include <mesh/ConnectivityDispatcher.hpp>

#include <mesh/ItemOfItemType.hpp>
#include <utils/CRSGraph.hpp>
#include <utils/Partitioner.hpp>

#include <iostream>

template <size_t Dimension>
template <ItemType item_type>
void
ConnectivityDispatcher<Dimension>::_buildNewOwner()
{
  std::cout << "#### " << rang::fgB::yellow << itemName(item_type) << rang::fg::reset << " _buildNewOwner() ####\n";

  if constexpr (item_type == ItemType::cell) {
    CRSGraph connectivity_graph = m_connectivity.ownCellToCellGraph();
    Partitioner P;
    Array new_owner_array = P.partition(connectivity_graph);

    std::cout << "new_owner_array = " << new_owner_array << '\n';

    const auto& cell_is_owned = m_connectivity.cellIsOwned();

    std::cout << "cell_is_owned = " << cell_is_owned << '\n';

    CellValue<int> cell_new_owner(m_connectivity);

    size_t i = 0;
    for (CellId cell_id = 0; cell_id < m_connectivity.numberOfCells(); ++cell_id) {
      if (cell_is_owned[cell_id]) {
        cell_new_owner[cell_id] = new_owner_array[i++];
      }
    }
    synchronize(cell_new_owner);

    std::cout << "cell_new_owner = " << cell_new_owner << '\n';
    std::cout << "cell_number    = " << m_connectivity.template number<item_type>() << '\n';

    this->_dispatchedInfo<ItemType::cell>().m_new_owner = cell_new_owner;
  } else {
    const auto& item_to_cell_matrix = m_connectivity.template getItemToItemMatrix<item_type, ItemType::cell>();

    const auto& cell_number    = m_connectivity.cellNumber();
    const auto& cell_new_owner = this->_dispatchedInfo<ItemType::cell>().m_new_owner;

    using ItemId = ItemIdT<item_type>;
    ItemValue<int, item_type> item_new_owner(m_connectivity);
    parallel_for(
      item_new_owner.numberOfItems(), PUGS_LAMBDA(const ItemId& l) {
        const auto& item_to_cell = item_to_cell_matrix[l];
        CellId Jmin              = item_to_cell[0];

        for (size_t j = 1; j < item_to_cell.size(); ++j) {
          const CellId J = item_to_cell[j];
          if (cell_number[J] < cell_number[Jmin]) {
            Jmin = J;
          }
        }
        item_new_owner[l] = cell_new_owner[Jmin];
      });

    synchronize(item_new_owner);

    std::cout << itemName(item_type) << "_new_owner = " << item_new_owner << '\n';
    std::cout << itemName(item_type) << "_number    = " << m_connectivity.template number<item_type>() << '\n';

    this->_dispatchedInfo<item_type>().m_new_owner = item_new_owner;
  }
}

template <size_t Dimension>
template <ItemType item_type>
void
ConnectivityDispatcher<Dimension>::_buildItemToExchangeLists()
{
  this->_buildItemListToSend<item_type>();
  this->_buildNumberOfItemToExchange<item_type>();
  if constexpr (item_type == ItemType::cell) {
    this->_buildCellNumberIdMap();
  }
  this->_buildRecvItemIdCorrespondanceByProc<item_type>();
}

template <size_t Dimension>
template <ItemType item_type>
void
ConnectivityDispatcher<Dimension>::_buildItemListToSend()
{
  if constexpr (item_type == ItemType::cell) {
    const auto& node_to_cell_matrix = m_connectivity.nodeToCellMatrix();
    const auto& cell_to_node_matrix = m_connectivity.cellToNodeMatrix();
    const auto& cell_is_owned       = m_connectivity.cellIsOwned();

    const auto& cell_new_owner = this->_dispatchedInfo<ItemType::cell>().m_new_owner;
    const auto& cell_number    = m_connectivity.cellNumber();

    std::cout << "- cell_new_owner = " << cell_new_owner << '\n';

    std::vector<std::vector<CellId>> cell_vector_to_send_by_proc(parallel::size());
    Array<bool> send_to_rank(parallel::size());

    for (CellId cell_id = 0; cell_id < m_connectivity.numberOfCells(); ++cell_id) {
      send_to_rank.fill(false);
      const auto& cell_to_node = cell_to_node_matrix[cell_id];

      std::cout << rang::fgB::cyan << " looking for cell " << cell_id << "(" << cell_number[cell_id] << ")"
                << rang::fg::reset << "\n";

      if (cell_is_owned[cell_id]) {
        for (size_t i_node = 0; i_node < cell_to_node.size(); ++i_node) {
          const NodeId& node_id    = cell_to_node[i_node];
          const auto& node_to_cell = node_to_cell_matrix[node_id];
          for (size_t i_node_cell = 0; i_node_cell < node_to_cell.size(); ++i_node_cell) {
            const CellId& node_cell                 = node_to_cell[i_node_cell];
            send_to_rank[cell_new_owner[node_cell]] = true;
            std::cout << rang::fgB::green << " storing cell " << node_cell << "(" << cell_number[node_cell] << ")"
                      << rang::fg::reset << "\n";
          }
        }

        for (size_t i_rank = 0; i_rank < send_to_rank.size(); ++i_rank) {
          if (send_to_rank[i_rank]) {
            cell_vector_to_send_by_proc[i_rank].push_back(cell_id);
          }
        }
      }
    }

    auto& cell_list_to_send_by_proc = this->_dispatchedInfo<ItemType::cell>().m_list_to_send_by_proc;
    cell_list_to_send_by_proc.resize(parallel::size());
    for (size_t i = 0; i < parallel::size(); ++i) {
      cell_list_to_send_by_proc[i] = convert_to_array(cell_vector_to_send_by_proc[i]);
    }

    for (size_t i = 0; i < parallel::size(); ++i) {
      std::cout << itemName(item_type) << " send to " << i << ": " << cell_list_to_send_by_proc[i] << '\n';
    }
  } else {
    const auto& cell_list_to_send_by_proc = this->_dispatchedInfo<ItemType::cell>().m_list_to_send_by_proc;

    const auto& item_is_owned = m_connectivity.template isOwned<item_type>();

    using ItemId                        = ItemIdT<item_type>;
    const auto& cell_to_sub_item_matrix = m_connectivity.template getItemToItemMatrix<ItemType::cell, item_type>();

    auto& item_list_to_send_by_proc = this->_dispatchedInfo<item_type>().m_list_to_send_by_proc;
    item_list_to_send_by_proc.resize(parallel::size());

    for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
      Array<bool> tag(m_connectivity.template numberOf<item_type>());
      tag.fill(false);
      std::vector<ItemId> item_id_vector;
      for (size_t j = 0; j < cell_list_to_send_by_proc[i_rank].size(); ++j) {
        const CellId& cell_id          = cell_list_to_send_by_proc[i_rank][j];
        const auto& cell_sub_item_list = cell_to_sub_item_matrix[cell_id];
        for (size_t r = 0; r < cell_sub_item_list.size(); ++r) {
          const ItemId& item_id = cell_sub_item_list[r];
          if (item_is_owned[item_id] and (not tag[item_id])) {
            item_id_vector.push_back(item_id);
            tag[item_id] = true;
          }
        }
      }
      item_list_to_send_by_proc[i_rank] = convert_to_array(item_id_vector);
    }

    for (size_t i = 0; i < parallel::size(); ++i) {
      std::cout << itemName(item_type) << " send to " << i << ": " << item_list_to_send_by_proc[i] << '\n';
    }
  }
}

template <size_t Dimension>
template <ItemType item_type>
void
ConnectivityDispatcher<Dimension>::_buildNumberOfItemToExchange()
{
  const auto& item_list_to_send_by_proc = this->_dispatchedInfo<item_type>().m_list_to_send_by_proc;
  Array<unsigned int> nb_item_to_send_by_proc(parallel::size());
  for (size_t i = 0; i < parallel::size(); ++i) {
    nb_item_to_send_by_proc[i] = item_list_to_send_by_proc[i].size();
  }
  this->_dispatchedInfo<item_type>().m_list_to_send_size_by_proc = nb_item_to_send_by_proc;

  std::cout << rang::fgB::yellow << itemName(item_type) << " nb_item_to_send_by_proc = " << nb_item_to_send_by_proc
            << rang::fg::reset << '\n';

  this->_dispatchedInfo<item_type>().m_list_to_recv_size_by_proc = parallel::allToAll(nb_item_to_send_by_proc);

  std::cout << rang::fgB::blue << itemName(item_type)
            << " m_list_to_recv_size_by_proc = " << this->_dispatchedInfo<item_type>().m_list_to_recv_size_by_proc
            << rang::fg::reset << '\n';
}

template <size_t Dimension>
template <typename DataType, ItemType item_type, typename ConnectivityPtr>
void
ConnectivityDispatcher<Dimension>::_gatherFrom(const ItemValue<DataType, item_type, ConnectivityPtr>& data_to_gather,
                                               Array<std::remove_const_t<DataType>>& gathered_array)
{
  std::vector<Array<const DataType>> recv_item_data_by_proc = this->exchange(data_to_gather);

  const auto& recv_id_correspondance_by_proc = this->_dispatchedInfo<item_type>().m_recv_id_correspondance_by_proc;
  Assert(recv_id_correspondance_by_proc.size() == parallel::size());

  gathered_array = Array<std::remove_const_t<DataType>>(this->_dispatchedInfo<item_type>().m_number_to_id_map.size());
  for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
    Assert(recv_id_correspondance_by_proc[i_rank].size() == recv_item_data_by_proc[i_rank].size());
    for (size_t r = 0; r < recv_id_correspondance_by_proc[i_rank].size(); ++r) {
      const auto& item_id     = recv_id_correspondance_by_proc[i_rank][r];
      gathered_array[item_id] = recv_item_data_by_proc[i_rank][r];
    }
  }
}

template <size_t Dimension>
template <typename DataType, typename ItemOfItem, typename ConnectivityPtr>
void
ConnectivityDispatcher<Dimension>::_gatherFrom(
  const SubItemValuePerItem<DataType, ItemOfItem, ConnectivityPtr>& data_to_gather,
  Array<std::remove_const_t<DataType>>& gathered_array)
{
  using MutableDataType = std::remove_const_t<DataType>;

  constexpr ItemType item_type = ItemOfItem::item_type;
  using ItemId                 = ItemIdT<item_type>;

  const auto& item_list_to_send_by_proc = this->_dispatchedInfo<item_type>().m_list_to_send_by_proc;

  std::vector<Array<MutableDataType>> data_to_send_by_proc(parallel::size());

  for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
    std::vector<MutableDataType> data_by_item_vector;
    for (size_t j = 0; j < item_list_to_send_by_proc[i_rank].size(); ++j) {
      const ItemId& item_id = item_list_to_send_by_proc[i_rank][j];
      const auto& item_data = data_to_gather.itemArray(item_id);
      for (size_t l = 0; l < item_data.size(); ++l) {
        data_by_item_vector.push_back(item_data[l]);
      }
    }
    data_to_send_by_proc[i_rank] = convert_to_array(data_by_item_vector);
  }

  const auto& number_of_sub_item_per_item_to_recv_by_proc =
    this->_dispatchedInfo<ItemOfItem>().m_number_of_sub_item_per_item_to_recv_by_proc;

  std::vector<Array<MutableDataType>> recv_data_to_gather_by_proc(parallel::size());
  for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
    recv_data_to_gather_by_proc[i_rank] =
      Array<MutableDataType>(sum(number_of_sub_item_per_item_to_recv_by_proc[i_rank]));
  }

  parallel::exchange(data_to_send_by_proc, recv_data_to_gather_by_proc);

  const size_t recv_array_size = [&] {
    size_t size = 0;
    for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
      size += recv_data_to_gather_by_proc[i_rank].size();
    }
    return size;
  }();

  gathered_array = Array<std::remove_const_t<DataType>>(recv_array_size);
  {
    size_t l = 0;
    for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
      for (size_t j = 0; j < recv_data_to_gather_by_proc[i_rank].size(); ++j) {
        gathered_array[l++] = recv_data_to_gather_by_proc[i_rank][j];
      }
    }
    Assert(gathered_array.size() == l);
  }
}

template <size_t Dimension>
void
ConnectivityDispatcher<Dimension>::_buildCellNumberIdMap()
{
  const auto recv_cell_number_by_proc = this->exchange(m_connectivity.template number<ItemType::cell>());
  auto& cell_number_id_map            = this->_dispatchedInfo<ItemType::cell>().m_number_to_id_map;
  for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
    CellId cell_id = 0;
    for (size_t i = 0; i < recv_cell_number_by_proc[i_rank].size(); ++i) {
      const int cell_number     = recv_cell_number_by_proc[i_rank][i];
      auto [iterator, inserted] = cell_number_id_map.insert(std::make_pair(cell_number, cell_id));
      if (inserted)
        ++cell_id;
    }
  }
}

template <size_t Dimension>
template <typename ItemOfItemT>
void
ConnectivityDispatcher<Dimension>::_buildSubItemNumberToIdMap()
{
  static_assert(ItemOfItemT::item_type == ItemType::cell, "Dispatcher requires to be built using cell as master "
                                                          "entities");

  const auto& cell_sub_item_number_to_recv_by_proc =
    this->_dispatchedInfo<ItemOfItemT>().m_sub_item_numbers_to_recv_by_proc;

  auto& sub_item_number_id_map = this->_dispatchedInfo<ItemOfItemT::sub_item_type>().m_number_to_id_map;
  for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
    int sub_item_id = 0;
    for (size_t i = 0; i < cell_sub_item_number_to_recv_by_proc[i_rank].size(); ++i) {
      int sub_item_number       = cell_sub_item_number_to_recv_by_proc[i_rank][i];
      auto [iterator, inserted] = sub_item_number_id_map.insert(std::make_pair(sub_item_number, sub_item_id));
      if (inserted)
        sub_item_id++;
    }
  }
}

template <size_t Dimension>
template <typename SubItemOfItemT>
void
ConnectivityDispatcher<Dimension>::_buildNumberOfSubItemPerItemToRecvByProc()
{
  const auto& item_to_sub_item_matrix =
    m_connectivity.template getItemToItemMatrix<SubItemOfItemT::item_type, SubItemOfItemT::sub_item_type>();

  ItemValue<int, SubItemOfItemT::item_type> number_of_sub_item_per_item(m_connectivity);

  using ItemId = ItemIdT<SubItemOfItemT::item_type>;
  parallel_for(
    number_of_sub_item_per_item.numberOfItems(),
    PUGS_LAMBDA(const ItemId& j) { number_of_sub_item_per_item[j] = item_to_sub_item_matrix[j].size(); });

  this->_dispatchedInfo<SubItemOfItemT>().m_number_of_sub_item_per_item_to_recv_by_proc =
    this->exchange(number_of_sub_item_per_item);
}

template <size_t Dimension>
template <typename SubItemOfItemT>
void
ConnectivityDispatcher<Dimension>::_buildSubItemNumbersToRecvByProc()
{
  const std::vector<Array<const int>> sub_item_numbers_to_send_by_proc = [&]() {
    const auto& item_to_sub_item_matrix =
      m_connectivity.template getItemToItemMatrix<SubItemOfItemT::item_type, SubItemOfItemT::sub_item_type>();

    const auto& sub_item_number = m_connectivity.template number<SubItemOfItemT::sub_item_type>();

    using ItemId    = ItemIdT<SubItemOfItemT::item_type>;
    using SubItemId = ItemIdT<SubItemOfItemT::sub_item_type>;

    std::vector<Array<const int>> mutable_sub_item_numbers_to_send_by_proc(parallel::size());
    for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
      const auto& item_list_to_send_by_proc = this->_dispatchedInfo<SubItemOfItemT::item_type>().m_list_to_send_by_proc;

      std::vector<int> sub_item_numbers_by_item_vector;
      for (size_t j = 0; j < item_list_to_send_by_proc[i_rank].size(); ++j) {
        const ItemId& item_id     = item_list_to_send_by_proc[i_rank][j];
        const auto& sub_item_list = item_to_sub_item_matrix[item_id];
        for (size_t r = 0; r < sub_item_list.size(); ++r) {
          const SubItemId& sub_item_id = sub_item_list[r];
          sub_item_numbers_by_item_vector.push_back(sub_item_number[sub_item_id]);
        }
      }
      mutable_sub_item_numbers_to_send_by_proc[i_rank] = convert_to_array(sub_item_numbers_by_item_vector);
    }
    return mutable_sub_item_numbers_to_send_by_proc;
  }();

  const auto& number_of_sub_item_per_item_to_recv_by_proc =
    this->_dispatchedInfo<SubItemOfItemT>().m_number_of_sub_item_per_item_to_recv_by_proc;

  std::vector<Array<int>> sub_item_numbers_to_recv_by_proc(parallel::size());
  for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
    sub_item_numbers_to_recv_by_proc[i_rank] = Array<int>(sum(number_of_sub_item_per_item_to_recv_by_proc[i_rank]));
  }
  parallel::exchange(sub_item_numbers_to_send_by_proc, sub_item_numbers_to_recv_by_proc);

  auto& const_sub_item_numbers_to_recv_by_proc =
    this->_dispatchedInfo<SubItemOfItemT>().m_sub_item_numbers_to_recv_by_proc;

  const_sub_item_numbers_to_recv_by_proc.resize(parallel::size());
  for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
    const_sub_item_numbers_to_recv_by_proc[i_rank] = sub_item_numbers_to_recv_by_proc[i_rank];
  }
}

template <size_t Dimension>
template <typename ItemOfItemT>
void
ConnectivityDispatcher<Dimension>::_buildItemToSubItemDescriptor()
{
  constexpr ItemType item_type     = ItemOfItemT::item_type;
  constexpr ItemType sub_item_type = ItemOfItemT::sub_item_type;

  const auto& item_list_to_recv_size_by_proc = this->_dispatchedInfo<item_type>().m_list_to_recv_size_by_proc;

  const auto& number_of_sub_item_per_item_to_recv_by_proc =
    this->_dispatchedInfo<ItemOfItemT>().m_number_of_sub_item_per_item_to_recv_by_proc;

  const auto& sub_item_number_id_map = this->_dispatchedInfo<sub_item_type>().m_number_to_id_map;

  const auto& recv_item_of_item_numbers_by_proc =
    this->_dispatchedInfo<ItemOfItemT>().m_sub_item_numbers_to_recv_by_proc;

  std::vector<std::vector<unsigned int>> item_to_subitem_legacy;
  size_t number_of_node_by_cell = 0;
  for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
    int l = 0;
    for (size_t i = 0; i < item_list_to_recv_size_by_proc[i_rank]; ++i) {
      std::vector<unsigned int> sub_item_vector;
      for (int k = 0; k < number_of_sub_item_per_item_to_recv_by_proc[i_rank][i]; ++k) {
        const auto& searched_sub_item_id = sub_item_number_id_map.find(recv_item_of_item_numbers_by_proc[i_rank][l++]);
        Assert(searched_sub_item_id != sub_item_number_id_map.end());
        sub_item_vector.push_back(searched_sub_item_id->second);
      }
      number_of_node_by_cell += sub_item_vector.size();

      item_to_subitem_legacy.emplace_back(sub_item_vector);
    }
  }
  Array<unsigned int> item_to_subitem_row_map(item_to_subitem_legacy.size() + 1);
  Array<unsigned int> item_to_subitem_list(number_of_node_by_cell);

  item_to_subitem_row_map.fill(10000000);
  item_to_subitem_list.fill(10000000);

  item_to_subitem_row_map[0] = 0;
  for (size_t i = 0; i < item_to_subitem_legacy.size(); ++i) {
    item_to_subitem_row_map[i + 1] = item_to_subitem_row_map[i] + item_to_subitem_legacy[i].size();
  }
  size_t l = 0;
  for (size_t i = 0; i < item_to_subitem_legacy.size(); ++i) {
    const auto& subitem_list = item_to_subitem_legacy[i];
    for (size_t j = 0; j < subitem_list.size(); ++j, ++l) {
      item_to_subitem_list[l] = subitem_list[j];
    }
  }

  m_new_descriptor.itemOfItemVector<ItemOfItemT>() = ConnectivityMatrix(item_to_subitem_row_map, item_to_subitem_list);
}

template <size_t Dimension>
template <ItemType item_type>
void
ConnectivityDispatcher<Dimension>::_buildRecvItemIdCorrespondanceByProc()
{
  const auto& item_list_to_send_by_proc = this->_dispatchedInfo<item_type>().m_list_to_send_by_proc;
  using ItemId                          = ItemIdT<item_type>;

  std::vector<Array<const ItemId>> recv_item_id_correspondance_by_proc(parallel::size());
  const ItemValue<const int, item_type>& item_number = m_connectivity.template number<item_type>();

  std::vector<Array<const int>> send_item_number_by_proc(parallel::size());
  for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
    Array<int> send_item_number(item_list_to_send_by_proc[i_rank].size());
    const Array<const ItemId> send_item_id = item_list_to_send_by_proc[i_rank];
    parallel_for(
      send_item_number.size(), PUGS_LAMBDA(size_t j) { send_item_number[j] = item_number[send_item_id[j]]; });
    send_item_number_by_proc[i_rank] = send_item_number;
  }

  const auto& item_list_to_recv_size_by_proc = this->_dispatchedInfo<item_type>().m_list_to_recv_size_by_proc;
  std::vector<Array<int>> recv_item_number_by_proc(parallel::size());
  for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
    recv_item_number_by_proc[i_rank] = Array<int>(item_list_to_recv_size_by_proc[i_rank]);
  }
  parallel::exchange(send_item_number_by_proc, recv_item_number_by_proc);

  const auto& item_number_to_id_map = this->_dispatchedInfo<item_type>().m_number_to_id_map;
  for (size_t i_rank = 0; i_rank < item_list_to_recv_size_by_proc.size(); ++i_rank) {
    Array<ItemId> item_id_correspondance(item_list_to_recv_size_by_proc[i_rank]);
    for (size_t l = 0; l < item_list_to_recv_size_by_proc[i_rank]; ++l) {
      const int& recv_item_number  = recv_item_number_by_proc[i_rank][l];
      const auto& searched_item_id = item_number_to_id_map.find(recv_item_number);
      Assert(searched_item_id != item_number_to_id_map.end());
      item_id_correspondance[l] = searched_item_id->second;
    }
    recv_item_id_correspondance_by_proc[i_rank] = item_id_correspondance;
  }
  this->_dispatchedInfo<item_type>().m_recv_id_correspondance_by_proc = recv_item_id_correspondance_by_proc;
}

template <size_t Dimension>
template <ItemType item_type>
void
ConnectivityDispatcher<Dimension>::_buildItemReferenceList()
{
  using ItemId = ItemIdT<item_type>;

  // Getting references
  Array<const size_t> number_of_item_ref_list_per_proc =
    parallel::allGather(m_connectivity.template numberOfRefItemList<item_type>());

  const size_t number_of_item_list_sender = [&]() {
    size_t mutable_number_of_item_list_sender = 0;
    for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
      mutable_number_of_item_list_sender += (number_of_item_ref_list_per_proc[i_rank] > 0);
    }
    return mutable_number_of_item_list_sender;
  }();

  if (number_of_item_list_sender > 0) {
    if (number_of_item_list_sender > 1) {
      std::cerr << __FILE__ << ':' << __LINE__ << ": " << rang::fgB::red
                << "need to check that knowing procs know the same item_ref_lists!" << rang::fg::reset << '\n';
    }

    if (number_of_item_list_sender < parallel::size()) {
      const size_t sender_rank = [&]() {
        size_t i_rank = 0;
        for (; i_rank < parallel::size(); ++i_rank) {
          if (number_of_item_ref_list_per_proc[i_rank] > 0) {
            break;
          }
        }
        return i_rank;
      }();

      Assert(number_of_item_list_sender < parallel::size());

      // sending is boundary property
      Array<RefItemListBase::Type> ref_item_list_type{number_of_item_ref_list_per_proc[sender_rank]};
      if (parallel::rank() == sender_rank) {
        for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
             ++i_item_ref_list) {
          auto item_ref_list                  = m_connectivity.template refItemList<item_type>(i_item_ref_list);
          ref_item_list_type[i_item_ref_list] = item_ref_list.type();
        }
      }
      parallel::broadcast(ref_item_list_type, sender_rank);

      // sending references tags
      Array<RefId::TagNumberType> ref_tag_list{number_of_item_ref_list_per_proc[sender_rank]};
      if (parallel::rank() == sender_rank) {
        for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
             ++i_item_ref_list) {
          auto item_ref_list            = m_connectivity.template refItemList<item_type>(i_item_ref_list);
          ref_tag_list[i_item_ref_list] = item_ref_list.refId().tagNumber();
        }
      }
      parallel::broadcast(ref_tag_list, sender_rank);

      // sending references name size
      Array<size_t> ref_name_size_list{number_of_item_ref_list_per_proc[sender_rank]};
      if (parallel::rank() == sender_rank) {
        for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
             ++i_item_ref_list) {
          auto item_ref_list                  = m_connectivity.template refItemList<item_type>(i_item_ref_list);
          ref_name_size_list[i_item_ref_list] = item_ref_list.refId().tagName().size();
        }
      }
      parallel::broadcast(ref_name_size_list, sender_rank);

      // sending references name size
      Array<RefId::TagNameType::value_type> ref_name_cat{sum(ref_name_size_list)};
      if (parallel::rank() == sender_rank) {
        size_t i_char = 0;
        for (size_t i_item_ref_list = 0; i_item_ref_list < m_connectivity.template numberOfRefItemList<item_type>();
             ++i_item_ref_list) {
          auto item_ref_list = m_connectivity.template refItemList<item_type>(i_item_ref_list);
          for (auto c : item_ref_list.refId().tagName()) {
            ref_name_cat[i_char++] = c;
          }
        }
      }
      parallel::broadcast(ref_name_cat, sender_rank);

      std::vector<RefId> ref_id_list = [&]() {
        std::vector<RefId> mutable_ref_id_list;
        mutable_ref_id_list.reserve(ref_name_size_list.size());
        size_t begining = 0;
        for (size_t i_ref = 0; i_ref < ref_name_size_list.size(); ++i_ref) {
          const size_t size = ref_name_size_list[i_ref];
          mutable_ref_id_list.emplace_back(ref_tag_list[i_ref], std::string{&(ref_name_cat[begining]), size});
          begining += size;
        }
        return mutable_ref_id_list;
      }();

      using block_type            = int32_t;
      constexpr size_t block_size = sizeof(block_type);
      const size_t nb_block       = ref_id_list.size() / block_size + (ref_id_list.size() % block_size != 0);
      for (size_t i_block = 0; i_block < nb_block; ++i_block) {
        ItemValue<block_type, item_type> item_references(m_connectivity);
        item_references.fill(0);

        if (m_connectivity.template numberOfRefItemList<item_type>() > 0) {
          const size_t max_i_ref = std::min(ref_id_list.size(), block_size * (i_block + 1));
          for (size_t i_ref = block_size * i_block, i = 0; i_ref < max_i_ref; ++i_ref, ++i) {
            block_type ref_bit{1 << i};
            auto item_ref_list = m_connectivity.template refItemList<item_type>(i_ref);

            const auto& item_list = item_ref_list.list();
            for (size_t i_item = 0; i_item < item_list.size(); ++i_item) {
              const ItemId& item_id = item_list[i_item];
              item_references[item_id] |= ref_bit;
            }
          }
        }

        const auto& nb_item_to_send_by_proc = this->_dispatchedInfo<item_type>().m_list_to_send_size_by_proc;

        const auto& send_item_id_by_proc = this->_dispatchedInfo<item_type>().m_list_to_send_by_proc;

        std::vector<Array<const block_type>> send_item_refs_by_proc(parallel::size());

        for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
          Array<block_type> send_item_refs(nb_item_to_send_by_proc[i_rank]);
          const Array<const ItemId> send_item_id = send_item_id_by_proc[i_rank];
          parallel_for(
            send_item_id.size(), PUGS_LAMBDA(size_t l) {
              const ItemId& item_id = send_item_id[l];
              send_item_refs[l]     = item_references[item_id];
            });
          send_item_refs_by_proc[i_rank] = send_item_refs;
        }

        std::vector<Array<block_type>> recv_item_refs_by_proc(parallel::size());
        for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
          recv_item_refs_by_proc[i_rank] =
            Array<block_type>(this->_dispatchedInfo<item_type>().m_list_to_recv_size_by_proc[i_rank]);
        }
        parallel::exchange(send_item_refs_by_proc, recv_item_refs_by_proc);

        const auto& recv_item_id_correspondance_by_proc =
          this->_dispatchedInfo<item_type>().m_recv_id_correspondance_by_proc;
        std::vector<block_type> item_refs(m_new_descriptor.template itemNumberVector<item_type>().size());
        for (size_t i_rank = 0; i_rank < parallel::size(); ++i_rank) {
          for (size_t r = 0; r < recv_item_refs_by_proc[i_rank].size(); ++r) {
            const ItemId& item_id = recv_item_id_correspondance_by_proc[i_rank][r];
            item_refs[item_id]    = recv_item_refs_by_proc[i_rank][r];
          }
        }

        const size_t max_i_ref = std::min(ref_id_list.size(), block_size * (i_block + 1));
        for (size_t i_ref = block_size * i_block, i = 0; i_ref < max_i_ref; ++i_ref, ++i) {
          block_type ref_bit{1 << i};

          std::vector<ItemId> item_id_vector;

          for (uint32_t i_item = 0; i_item < item_refs.size(); ++i_item) {
            const ItemId item_id{i_item};
            if (item_refs[item_id] & ref_bit) {
              item_id_vector.push_back(item_id);
            }
          }

          Array<const ItemId> item_id_array = convert_to_array(item_id_vector);

          RefItemListBase::Type type = ref_item_list_type[i_ref];
          m_new_descriptor.addRefItemList(RefItemList<item_type>(ref_id_list[i_ref], item_id_array, type));
        }
      }
    }
  }
}

template <size_t Dimension>
void
ConnectivityDispatcher<Dimension>::_dispatchEdges()
{
  if constexpr (Dimension > 2) {
    this->_buildNumberOfSubItemPerItemToRecvByProc<EdgeOfCell>();
    this->_buildSubItemNumbersToRecvByProc<EdgeOfCell>();
    this->_buildSubItemNumberToIdMap<EdgeOfCell>();
    this->_buildItemToExchangeLists<ItemType::edge>();

    m_new_descriptor.setEdgeNumberVector([&] {
      Array<int> edge_number_vector;
      this->_gatherFrom(m_connectivity.template number<ItemType::edge>(), edge_number_vector);
      return edge_number_vector;
    }());

    this->_buildItemToSubItemDescriptor<EdgeOfCell>();

    this->_buildNumberOfSubItemPerItemToRecvByProc<NodeOfEdge>();
    this->_buildSubItemNumbersToRecvByProc<NodeOfEdge>();
    this->_buildItemToSubItemDescriptor<NodeOfEdge>();

    this->_buildNumberOfSubItemPerItemToRecvByProc<EdgeOfFace>();
    this->_buildSubItemNumbersToRecvByProc<EdgeOfFace>();
    this->_buildItemToSubItemDescriptor<EdgeOfFace>();

    m_new_descriptor.setFaceEdgeIsReversed([&] {
      Array<bool> face_edge_is_reversed;
      this->_gatherFrom(m_connectivity.faceEdgeIsReversed(), face_edge_is_reversed);
      return face_edge_is_reversed;
    }());

    m_new_descriptor.setEdgeOwnerVector([&] {
      Array<int> edge_owner_vector;
      this->_gatherFrom(this->_dispatchedInfo<ItemType::edge>().m_new_owner, edge_owner_vector);
      return edge_owner_vector;
    }());

    this->_buildItemReferenceList<ItemType::edge>();
  }
}

template <size_t Dimension>
void
ConnectivityDispatcher<Dimension>::_dispatchFaces()
{
  if constexpr (Dimension > 1) {
    this->_buildNumberOfSubItemPerItemToRecvByProc<FaceOfCell>();
    this->_buildSubItemNumbersToRecvByProc<FaceOfCell>();
    this->_buildSubItemNumberToIdMap<FaceOfCell>();
    this->_buildItemToExchangeLists<ItemType::face>();

    this->_buildNumberOfSubItemPerItemToRecvByProc<NodeOfFace>();
    this->_buildSubItemNumbersToRecvByProc<NodeOfFace>();
    this->_buildItemToSubItemDescriptor<NodeOfFace>();

    m_new_descriptor.setFaceNumberVector([&] {
      Array<int> face_number_vector;
      this->_gatherFrom(m_connectivity.template number<ItemType::face>(), face_number_vector);
      return face_number_vector;
    }());

    this->_buildItemToSubItemDescriptor<FaceOfCell>();

    m_new_descriptor.setCellFaceIsReversed([&] {
      Array<bool> cell_face_is_reversed;
      this->_gatherFrom(m_connectivity.cellFaceIsReversed(), cell_face_is_reversed);
      return cell_face_is_reversed;
    }());

    m_new_descriptor.setFaceOwnerVector([&] {
      Array<int> face_owner_vector;
      this->_gatherFrom(this->_dispatchedInfo<ItemType::face>().m_new_owner, face_owner_vector);
      return face_owner_vector;
    }());

    this->_buildItemReferenceList<ItemType::face>();
  }
}

template <size_t Dimension>
ConnectivityDispatcher<Dimension>::ConnectivityDispatcher(const ConnectivityType& connectivity)
  : m_connectivity(connectivity)
{
  {
    Array connectivity_id_list = parallel::allGather(connectivity.id());
    if (min(connectivity_id_list) != max(connectivity_id_list)) {
      throw UnexpectedError("connectivity ids diverged in parallel");
    }
  }

  std::cout << "--- initial mesh ---\n";
  std::cout << "--- cells\n";
  for (CellId cell_id = 0; cell_id < connectivity.numberOfCells(); ++cell_id) {
    std::cout << "cell " << cell_id << "(" << connectivity.cellNumber()[cell_id] << "):" << rang::style::bold
              << ((connectivity.cellIsOwned()[cell_id]) ? 'O' : 'G') << rang::style::reset << ": ";
    auto node_list = connectivity.cellToNodeMatrix()[cell_id];
    for (size_t i_node = 0; i_node < node_list.size(); ++i_node) {
      const NodeId node_id = node_list[i_node];
      std::cout << ' ' << node_id << "(" << connectivity.nodeNumber()[node_id] << "):" << rang::style::bold
                << ((connectivity.nodeIsOwned()[node_id]) ? 'O' : 'G') << rang::style::reset;
    }
    std::cout << '\n';
  }
  if constexpr (Dimension > 1) {
    std::cout << "--- faces\n";
    for (FaceId face_id = 0; face_id < connectivity.numberOfFaces(); ++face_id) {
      std::cout << "cell " << face_id << "(" << connectivity.faceNumber()[face_id] << "):" << rang::style::bold
                << ((connectivity.faceIsOwned()[face_id]) ? 'O' : 'G') << rang::style::reset << ": ";
      auto node_list = connectivity.faceToNodeMatrix()[face_id];
      for (size_t i_node = 0; i_node < node_list.size(); ++i_node) {
        const NodeId node_id = node_list[i_node];
        std::cout << ' ' << node_id << "(" << connectivity.nodeNumber()[node_id] << "):" << rang::style::bold
                  << ((connectivity.nodeIsOwned()[node_id]) ? 'O' : 'G') << rang::style::reset;
      }
      std::cout << '\n';
    }
  }
  std::cout << "--- nodes\n";
  for (NodeId node_id = 0; node_id < connectivity.numberOfNodes(); ++node_id) {
    std::cout << "node " << node_id << "(" << connectivity.nodeNumber()[node_id] << "):" << rang::style::bold
              << ((connectivity.nodeIsOwned()[node_id]) ? 'O' : 'G') << rang::style::reset << ": ";
    std::cout << '\n';
  }

  this->_buildNewOwner<ItemType::cell>();
  if constexpr (Dimension > 1) {
    this->_buildNewOwner<ItemType::face>();
  }
  if constexpr (Dimension > 2) {
    this->_buildNewOwner<ItemType::edge>();
  }
  this->_buildNewOwner<ItemType::node>();

  this->_buildItemToExchangeLists<ItemType::cell>();

  this->_buildNumberOfSubItemPerItemToRecvByProc<NodeOfCell>();

  this->_buildSubItemNumbersToRecvByProc<NodeOfCell>();

  m_new_descriptor.setCellNumberVector([&] {
    Array<int> cell_number_vector;
    this->_gatherFrom(m_connectivity.template number<ItemType::cell>(), cell_number_vector);
    return cell_number_vector;
  }());

  this->_buildSubItemNumberToIdMap<NodeOfCell>();

  this->_buildItemToExchangeLists<ItemType::node>();

  m_new_descriptor.setCellTypeVector([&] {
    Array<CellType> cell_type_vector;
    this->_gatherFrom(m_connectivity.cellType(), cell_type_vector);
    return cell_type_vector;
  }());

  m_new_descriptor.setCellOwnerVector([&] {
    Array<int> cell_owner_vector;
    this->_gatherFrom(this->_dispatchedInfo<ItemType::cell>().m_new_owner, cell_owner_vector);
    return cell_owner_vector;
  }());

  m_new_descriptor.setNodeNumberVector([&] {
    Array<int> node_number_vector;
    this->_gatherFrom(m_connectivity.template number<ItemType::node>(), node_number_vector);
    return node_number_vector;
  }());

  m_new_descriptor.setNodeOwnerVector([&] {
    Array<int> node_owner_vector;
    this->_gatherFrom(this->_dispatchedInfo<ItemType::node>().m_new_owner, node_owner_vector);
    return node_owner_vector;
  }());

  this->_buildItemToSubItemDescriptor<NodeOfCell>();

  this->_buildItemReferenceList<ItemType::cell>();

  this->_dispatchFaces();

  this->_dispatchEdges();

  this->_buildItemReferenceList<ItemType::node>();

  std::cout << "m_new_descriptor.nodeNumberVector() = " << m_new_descriptor.nodeNumberVector() << '\n';
  std::cout << "m_new_descriptor.nodeOwnerVector()  = " << m_new_descriptor.nodeOwnerVector() << '\n';

  std::cout << "m_new_descriptor.cellOwnerVector().size()          = " << m_new_descriptor.cellOwnerVector().size()
            << '\n';
  std::cout << "m_new_descriptor.cellTypeVector().size()           = " << m_new_descriptor.cellTypeVector().size()
            << '\n';
  std::cout << "m_new_descriptor.cellToNodeMatrix().numberOfRows() = "
            << m_new_descriptor.cellToNodeMatrix().numberOfRows() << '\n';

  for (CellId cell_id = 0; cell_id < m_new_descriptor.cellToNodeMatrix().numberOfRows(); ++cell_id) {
    std::cout << "- cell [" << cell_id << "]("   // << m_new_descriptor.cellNumberVector()[cell_id]
              << ")" << m_new_descriptor.cellToNodeMatrix()[cell_id] << '\n';
  }

  std::cout.flush();
  parallel::barrier();

  m_dispatched_connectivity = ConnectivityType::build(m_new_descriptor);

  {
    Array connectivity_id_list = parallel::allGather(m_dispatched_connectivity->id());
    if (min(connectivity_id_list) != max(connectivity_id_list)) {
      throw UnexpectedError("connectivity ids diverged in parallel");
    }
  }
}

template ConnectivityDispatcher<1>::ConnectivityDispatcher(const ConnectivityType&);
template ConnectivityDispatcher<2>::ConnectivityDispatcher(const ConnectivityType&);
template ConnectivityDispatcher<3>::ConnectivityDispatcher(const ConnectivityType&);