Use new fillpatcher (#551)

Source/Advance.cpp

@@ -83,7 +83,7 @@ PeleC::do_mol_advance(
   if (verbose != 0) {
     amrex::Print() << "... Computing MOL source term at t^{n} " << std::endl;
   }
-  FillPatch(*this, Sborder, numGrow() + nGrowF, time, State_Type, 0, NVAR);
+  FillPatcherFill(Sborder, 0, NVAR, numGrow() + nGrowF, time, State_Type, 0);
   amrex::Real flux_factor = 0;
   getMOLSrcTerm(Sborder, molSrc, time, dt, flux_factor);
 
@@ -118,7 +118,8 @@ PeleC::do_mol_advance(
   if (verbose != 0) {
     amrex::Print() << "... Computing MOL source term at t^{n+1} " << std::endl;
   }
-  FillPatch(*this, Sborder, numGrow() + nGrowF, time + dt, State_Type, 0, NVAR);
+  FillPatcherFill(
+    Sborder, 0, NVAR, numGrow() + nGrowF, time + dt, State_Type, 0);
   flux_factor = mol_iters > 1 ? 0 : 1;
   getMOLSrcTerm(Sborder, molSrc, time, dt, flux_factor);
 
@@ -164,8 +165,8 @@ PeleC::do_mol_advance(
         amrex::Print() << "... Re-computing MOL source term at t^{n+1} (iter = "
                        << mol_iter << " of " << mol_iters << ")" << std::endl;
       }
-      FillPatch(
-        *this, Sborder, numGrow() + nGrowF, time + dt, State_Type, 0, NVAR);
+      FillPatcherFill(
+        Sborder, 0, NVAR, numGrow() + nGrowF, time + dt, State_Type, 0);
       flux_factor = mol_iter == mol_iters ? 1 : 0;
       getMOLSrcTerm(Sborder, molSrc_new, time, dt, flux_factor);
 
@@ -281,7 +282,7 @@ PeleC::do_sdc_iteration(
 #endif
 
   if (fill_Sborder) {
-    FillPatch(*this, Sborder, nGrow_Sborder, time, State_Type, 0, NVAR);
+    FillPatcherFill(Sborder, 0, NVAR, nGrow_Sborder, time, State_Type, 0);
   }
 
   if (sub_iteration == 0) {
@@ -409,7 +410,7 @@ PeleC::do_sdc_iteration(
       amrex::Print() << "... Computing diffusion terms at t^(n+1,"
                      << sub_iteration + 1 << ")" << std::endl;
     }
-    FillPatch(*this, Sborder, numGrow(), time + dt, State_Type, 0, NVAR);
+    FillPatcherFill(Sborder, 0, NVAR, numGrow(), time + dt, State_Type, 0);
     amrex::Real flux_factor_new = sub_iteration == sub_ncycle - 1 ? 0.5 : 0;
     getMOLSrcTerm(Sborder, *new_sources[diff_src], time, dt, flux_factor_new);
   }
@@ -430,7 +431,8 @@ PeleC::do_sdc_iteration(
       amrex::Print() << "moveKick ... updating velocity only\n";
 
     if (!do_diffuse) { // Else, this was already done above.  No need to redo
-      FillPatch(*this, Sborder, nGrow_Sborder, time + dt, State_Type, 0, NVAR);
+      FillPatcherFill(
+        Sborder, 0, NVAR, nGrow_Sborder, time + dt, State_Type, 0);
     }
 
     new_sources[spray_src]->setVal(0.);

Source/Diffusion.cpp

@@ -540,14 +540,14 @@ PeleC::getMOLSrcTerm(
             });
         }
 
-        if (level < parent->finestLevel()) {
-          getFluxReg(level + 1).CrseAdd(
+        if ((level < parent->finestLevel()) && (fr_as_crse != nullptr)) {
+          fr_as_crse->CrseAdd(
             mfi, {{AMREX_D_DECL(&flux_ec[0], &flux_ec[1], &flux_ec[2])}},
             dxD.data(), dt, amrex::RunOn::Device);
         }
 
-        if (level > 0) {
-          getFluxReg(level).FineAdd(
+        if ((level > 0) && (fr_as_fine != nullptr)) {
+          fr_as_fine->FineAdd(
             mfi, {{AMREX_D_DECL(&flux_ec[0], &flux_ec[1], &flux_ec[2])}},
             dxD.data(), dt, amrex::RunOn::Device);
         }

Source/IO.cpp

@@ -199,7 +199,7 @@ PeleC::restart(amrex::Amr& papa, std::istream& is, bool bReadSpecial)
   */
 
   if (level > 0 && do_reflux) {
-    flux_reg.define(
+    flux_reg = std::make_unique<amrex::EBFluxRegister>(
       grids, papa.boxArray(level - 1), dmap, papa.DistributionMap(level - 1),
       geom, papa.Geom(level - 1), papa.refRatio(level - 1), level, NVAR);
 

Source/PeleC.H

@@ -520,8 +520,8 @@ public:
 
   static bool eb_in_domain;
 
-  amrex::EBFluxRegister flux_reg;
-  amrex::EBFluxRegister& getFluxReg();
+  std::unique_ptr<amrex::EBFluxRegister> flux_reg;
+  amrex::EBFluxRegister& getFluxReg() const;
   amrex::EBFluxRegister& getFluxReg(int lev);
 
   static bool ebInitialized();
@@ -659,8 +659,6 @@ protected:
   amrex::MultiFab mms_source;
 #endif
 
-  // bool FillPatchedOldState_ok;
-
   // There can be only one Diffusion object, it covers all levels:
   static class Diffusion* diffusion;
 
@@ -810,9 +808,9 @@ pc_check_initial_species(
 
 AMREX_FORCE_INLINE
 amrex::EBFluxRegister&
-PeleC::getFluxReg()
+PeleC::getFluxReg() const
 {
-  return flux_reg;
+  return *flux_reg;
 }
 
 AMREX_FORCE_INLINE

Source/PeleC.cpp

@@ -417,7 +417,7 @@ PeleC::PeleC(
   }
 
   if (do_reflux && level > 0) {
-    flux_reg.define(
+    flux_reg = std::make_unique<amrex::EBFluxRegister>(
       bl, papa.boxArray(level - 1), dm, papa.DistributionMap(level - 1),
       level_geom, papa.Geom(level - 1), papa.refRatio(level - 1), level, NVAR);
 
@@ -1014,6 +1014,10 @@ PeleC::post_timestep(int
     // Clean up any aberrant state data generated by the reflux.
     // amrex::MultiFab& S_new_crse = get_new_data(State_Type);
     // clean_state(S_new_crse);
+
+    // fillpatcher on level+1 needs to be reset because data on this
+    // level have changed.
+    getLevel(level + 1).resetFillPatcher();
   }
 
   // Re-compute temperature after all the other updates.
@@ -1230,7 +1234,7 @@ PeleC::reflux()
   PeleC& fine_level = getLevel(level + 1);
   amrex::MultiFab& S_crse = get_new_data(State_Type);
   amrex::MultiFab& S_fine = fine_level.get_new_data(State_Type);
-  fine_level.flux_reg.Reflux(S_crse, vfrac, S_fine, fine_level.vfrac);
+  getFluxReg(level + 1).Reflux(S_crse, vfrac, S_fine, fine_level.vfrac);
 
   if (!amrex::DefaultGeometry().IsCartesian() && eb_in_domain) {
     amrex::Abort("rz not yet compatible with EB");