Skip to content

Make local an IntVect for Redistribute #5520

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
atmyers wants to merge 1 commit into
base: development
Choose a base branch
from
Open

Make local an IntVect for Redistribute #5520

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 9 additions & 3 deletions Src/Particle/AMReX_NeighborParticles.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -349,19 +349,25 @@ public:
calcCommSize();
}

void Redistribute (int lev_min=0, int lev_max=-1, int nGrow=0, int local=0,
bool remove_negative=true)
void Redistribute (int lev_min=0, int lev_max=-1, int nGrow=0,
IntVect local=IntVect(0), bool remove_negative=true)
{
clearNeighbors();
ParticleContainerType::Redistribute(lev_min, lev_max, nGrow, local, remove_negative);
}

void Redistribute (int lev_min, int lev_max, int nGrow, int local,
bool remove_negative=true)
{
Redistribute(lev_min, lev_max, nGrow, IntVect(local), remove_negative);
}

void RedistributeLocal (bool remove_negative=true)
{
const int lev_min = 0;
const int lev_max = 0;
const int nGrow = 0;
const int local = 1;
const IntVect local(1);
clearNeighbors();
this->Redistribute(lev_min, lev_max, nGrow, local, remove_negative);
}
Expand Down
17 changes: 14 additions & 3 deletions Src/Particle/AMReX_ParticleCommunication.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -410,16 +410,16 @@ public:
const ParticleCopyOp& op,
const Vector<int>& int_comp_mask,
const Vector<int>& real_comp_mask,
int local)
IntVect local)
{
BL_PROFILE("ParticleCopyPlan::build");

ParmParse pp("particles");
pp.query("do_one_sided_comms", m_do_one_sided_comms);
const int num_buckets = pc.BufferMap().numBuckets();

m_local = local;
if (local)
m_local = local.allGT(0);
if (m_local)
{
m_neighbor_procs = pc.NeighborProcs(local);
}
Expand Down Expand Up @@ -507,6 +507,17 @@ public:
buildMPIStart(pc, pc.BufferMap(), m_superparticle_size);
}

template <class PC>
requires (IsParticleContainer<PC>::value)
void build (const PC& pc,
const ParticleCopyOp& op,
const Vector<int>& int_comp_mask,
const Vector<int>& real_comp_mask,
int local)
{
build(pc, op, int_comp_mask, real_comp_mask, IntVect(local));
}

void clear ();

void buildMPIFinish (const ParticleBufferMap& map);
Expand Down
58 changes: 47 additions & 11 deletions Src/Particle/AMReX_ParticleContainer.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -504,14 +504,21 @@ public:
* redistributed but are not necessarily at the same time as those on the coarse
* level.
* Default: 0
* \param local If 0, this will be a non-local redistribute, meaning that particle can potentially
* go to any other box in the simulation. If > 0, this is the maximum number of cells
* a particle can have moved since the last Redistribute() call. Knowing this number
* allows an optimized MPI communication pattern to be used.
* \param local If all components are 0, this will be a non-local redistribute, meaning that
* particles can potentially go to any other box in the simulation. Otherwise,
* each component is the maximum number of cells a particle can have moved in
* that direction since the last Redistribute() call. Knowing this allows an
* optimized MPI communication pattern to be used.
* \param remove_negative If true, remove particles with a negative ID. Default: true.
*/
void Redistribute (int lev_min = 0, int lev_max = -1, int nGrow = 0, int local=0,
bool remove_negative=true);
void Redistribute (int lev_min = 0, int lev_max = -1, int nGrow = 0,
IntVect local=IntVect(0), bool remove_negative=true);

void Redistribute (int lev_min, int lev_max, int nGrow, int local,
bool remove_negative=true)
{
Redistribute(lev_min, lev_max, nGrow, IntVect(local), remove_negative);
}


/**
Expand Down Expand Up @@ -1279,11 +1286,23 @@ public:
return doUnlink;
}

void RedistributeCPU (int lev_min = 0, int lev_max = -1, int nGrow = 0, int local=0,
bool remove_negative=true);
void RedistributeCPU (int lev_min = 0, int lev_max = -1, int nGrow = 0,
IntVect local=IntVect(0), bool remove_negative=true);

void RedistributeCPU (int lev_min, int lev_max, int nGrow, int local,
bool remove_negative=true)
{
RedistributeCPU(lev_min, lev_max, nGrow, IntVect(local), remove_negative);
}

void Redistribute_impl (int lev_min = 0, int lev_max = -1, int nGrow = 0, int local=0,
bool remove_negative=true);
void Redistribute_impl (int lev_min = 0, int lev_max = -1, int nGrow = 0,
IntVect local=IntVect(0), bool remove_negative=true);

void Redistribute_impl (int lev_min, int lev_max, int nGrow, int local,
bool remove_negative=true)
{
Redistribute_impl(lev_min, lev_max, nGrow, IntVect(local), remove_negative);
}

void ReserveForRedistribute (ParticleCopyPlan const& plan);

Expand Down Expand Up @@ -1548,14 +1567,31 @@ private:
AMREX_FORCE_INLINE
int hostPartitionTile (ParticleTileType& src_tile,
int lev, int gid, int tid,
int lev_min, int lev_max, int nGrow, int local,
int lev_min, int lev_max, int nGrow, IntVect local,
bool remove_negative, int myproc,
Gpu::DeviceVector<int>& boxes,
Gpu::DeviceVector<int>& levels,
Gpu::DeviceVector<int>& tiles,
Gpu::DeviceVector<int>& src_indices,
Gpu::DeviceVector<IntVect>& periodic_shift);

AMREX_FORCE_INLINE
int hostPartitionTile (ParticleTileType& src_tile,
int lev, int gid, int tid,
int lev_min, int lev_max, int nGrow, int local,
bool remove_negative, int myproc,
Gpu::DeviceVector<int>& boxes,
Gpu::DeviceVector<int>& levels,
Gpu::DeviceVector<int>& tiles,
Gpu::DeviceVector<int>& src_indices,
Gpu::DeviceVector<IntVect>& periodic_shift)
{
return hostPartitionTile(src_tile, lev, gid, tid,
lev_min, lev_max, nGrow, IntVect(local),
remove_negative, myproc,
boxes, levels, tiles, src_indices, periodic_shift);
}

template <typename P>
void locateParticle (P& p, ParticleLocData& pld,
int lev_min, int lev_max, int nGrow, int local_grid=-1) const;
Expand Down
16 changes: 13 additions & 3 deletions Src/Particle/AMReX_ParticleContainerBase.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,11 +265,16 @@ public:
{ return m_particle_handshake_window ? m_particle_handshake_window->win : MPI_WIN_NULL; }
#endif

Vector<int> NeighborProcs(int ngrow) const
Vector<int> NeighborProcs(IntVect ngrow) const
{
return computeNeighborProcs(this->GetParGDB(), ngrow);
}

Vector<int> NeighborProcs(int ngrow) const
{
return NeighborProcs(IntVect(ngrow));
}

template <class MF>
bool OnSameGrids (int level, const MF& mf) const { return m_gdb->OnSameGrids(level, mf); }

Expand Down Expand Up @@ -297,7 +302,12 @@ public:

protected:

void BuildRedistributeMask (int lev, int nghost=1) const;
void BuildRedistributeMask (int lev, IntVect nghost) const;

void BuildRedistributeMask (int lev, int nghost=1) const
{
BuildRedistributeMask(lev, IntVect(nghost));
}
void defineBufferMap () const;

int m_verbose{0};
Expand All @@ -308,7 +318,7 @@ protected:
Arena* m_arena = nullptr;

mutable std::unique_ptr<iMultiFab> redistribute_mask_ptr;
mutable int redistribute_mask_nghost = std::numeric_limits<int>::min();
mutable IntVect redistribute_mask_nghost = IntVect(std::numeric_limits<int>::min());
mutable amrex::Vector<int> neighbor_procs;
mutable ParticleBufferMap m_buffer_map;

Expand Down
4 changes: 2 additions & 2 deletions Src/Particle/AMReX_ParticleContainerBase.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -329,13 +329,13 @@ int ParticleContainerBase::AggregationBuffer ()
return aggregation_buffer;
}

void ParticleContainerBase::BuildRedistributeMask (int lev, int nghost) const
void ParticleContainerBase::BuildRedistributeMask (int lev, IntVect nghost) const
{
BL_PROFILE("ParticleContainer::BuildRedistributeMask");
AMREX_ASSERT(lev == 0);

if (redistribute_mask_ptr == nullptr ||
redistribute_mask_nghost < nghost ||
! redistribute_mask_nghost.allGE(nghost) ||
! BoxArray::SameRefs(redistribute_mask_ptr->boxArray(), this->ParticleBoxArray(lev)) ||
! DistributionMapping::SameRefs(redistribute_mask_ptr->DistributionMap(), this->ParticleDistributionMap(lev)))
{
Expand Down
12 changes: 6 additions & 6 deletions Src/Particle/AMReX_ParticleContainerI.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1236,7 +1236,7 @@ template <typename ParticleType, int NArrayReal, int NArrayInt,
template<class> class Allocator, class CellAssignor>
void
ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor>
::Redistribute (int lev_min, int lev_max, int nGrow, int local, bool remove_negative)
::Redistribute (int lev_min, int lev_max, int nGrow, IntVect local, bool remove_negative)
{
BL_PROFILE_SYNC_START_TIMED("SyncBeforeComms: Redist");

Expand Down Expand Up @@ -1353,7 +1353,7 @@ int
ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor>
::hostPartitionTile (ParticleTileType& src_tile,
int lev, int gid, int tid,
int lev_min, int lev_max, int nGrow, int local,
int lev_min, int lev_max, int nGrow, IntVect local,
bool remove_negative, int myproc,
Gpu::DeviceVector<int>& boxes,
Gpu::DeviceVector<int>& levels,
Expand Down Expand Up @@ -1395,7 +1395,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
}

// full locate
locateParticle(p, pld, lev_min, lev_max, nGrow, local ? gid : -1);
locateParticle(p, pld, lev_min, lev_max, nGrow, local.allGT(0) ? gid : -1);
particlePostLocate(p, pld, lev);

// particle may have been invalidated, so check if we need to remove again.
Expand Down Expand Up @@ -1440,9 +1440,9 @@ template <typename ParticleType, int NArrayReal, int NArrayInt,
template<class> class Allocator, class CellAssignor>
void
ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssignor>
::Redistribute_impl (int lev_min, int lev_max, int nGrow, int local, bool remove_negative)
::Redistribute_impl (int lev_min, int lev_max, int nGrow, IntVect local, bool remove_negative)
{
if (local) { AMREX_ASSERT(numParticlesOutOfRange(*this, lev_min, lev_max, local) == 0); }
if (local.allGT(0)) { AMREX_ASSERT(numParticlesOutOfRange(*this, lev_min, lev_max, local) == 0); }

BL_PROFILE("ParticleContainer::Redistribute_impl()");
BL_PROFILE_VAR_NS("Redistribute_partition", blp_partition);
Expand Down Expand Up @@ -1474,7 +1474,7 @@ ParticleContainer_impl<ParticleType, NArrayReal, NArrayInt, Allocator, CellAssig
this->defineBufferMap();

#ifndef AMREX_USE_GPU
if (local > 0) { BuildRedistributeMask(0, local); }
if (local.allGT(0)) { BuildRedistributeMask(0, local); }
#else
if (! m_particle_locator.isValid(GetParGDB(), do_tiling, tile_size)) { m_particle_locator.build(GetParGDB(), do_tiling, tile_size); }
m_particle_locator.setGeometry(GetParGDB());
Expand Down
2 changes: 2 additions & 0 deletions Src/Particle/AMReX_ParticleUtil.H
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -747,6 +747,8 @@ IntVect computeRefFac (const ParGDBBase* a_gdb, int src_lev, int lev);

Vector<int> computeNeighborProcs (const ParGDBBase* a_gdb, int ngrow);

Vector<int> computeNeighborProcs (const ParGDBBase* a_gdb, IntVect ngrow);

/// \cond DOXYGEN_IGNORE
namespace particle_detail
{
Expand Down
7 changes: 6 additions & 1 deletion Src/Particle/AMReX_ParticleUtil.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ IntVect computeRefFac (const ParGDBBase* a_gdb, int src_lev, int lev)
return ref_fac;
}

Vector<int> computeNeighborProcs (const ParGDBBase* a_gdb, int ngrow)
Vector<int> computeNeighborProcs (const ParGDBBase* a_gdb, IntVect ngrow)
{
BL_PROFILE("amrex::computeNeighborProcs");

Expand Down Expand Up @@ -64,6 +64,11 @@ Vector<int> computeNeighborProcs (const ParGDBBase* a_gdb, int ngrow)
RemoveDuplicates(neighbor_procs);
return neighbor_procs;
}

Vector<int> computeNeighborProcs (const ParGDBBase* a_gdb, int ngrow)
{
return computeNeighborProcs(a_gdb, IntVect(ngrow));
}
}

#ifdef AMREX_USE_HDF5_ASYNC
Expand Down
Loading