eschnett/cactusamrex - Change MSBBCXVGD3GRLE5KAI6BKAFRV7SQUWI2SNN43AJAUD3ISRCEXY6QC

Snapshot before transitioning from AmrMesh to AmrCore

Created by Erik Schnetter on July 18, 2019

MSBBCXVGD3GRLE5KAI6BKAFRV7SQUWI2SNN43AJAUD3ISRCEXY6QC

Dependencies

In channels

master

main

Change contents

Insertion in AMReX/interface.ccl at line 4 [3.17976]

[3.18035]


PUBLIC:
CCTK_REAL regrid_tag TYPE=gf
{
  tag
} "Regridding tags"

Insertion in AMReX/src/driver.cxx at line 44 [3.15972]

[4.82]

[5.0]


namespace {
// Convert a (direction, face) pair to an AMReX Orientation
Orientation orient(int d, int f) {
  return Orientation(d, Orientation::Side(f));
}
} // namespace

Replacement in AMReX/src/driver.cxx at line 67 [3.15972]

B:BD[6.316] → [5.723:804]

void CactusAmrMesh::ErrorEst(int lev, TagBoxArray &tags, Real time, int ngrow) {

[6.316]

[5.804]

void CactusAmrMesh::ErrorEst(const int level, TagBoxArray &tags, Real time,
                             int ngrow) {
  // Don't regrid before Cactus is ready to
  if (level >= int(ghext->leveldata.size()))
    return;
  CCTK_VINFO("ErrorEst level %d", level);

Replacement in AMReX/src/driver.cxx at line 76 [3.15972]

B:BD[5.830] → [5.830:876]

  // tags.setVal(boxArray(lev), TagBox::SET);

[5.830]

[5.876]

  // tags.setVal(boxArray(level), TagBox::SET);

Replacement in AMReX/src/driver.cxx at line 79 [3.15972]
B:BD[5.896] → [7.0:39]
```
  const Box &dom = Geom(lev).Domain();
```
[5.896]
[5.969]
```
  const Box &dom = Geom(level).Domain();
```

Replacement in AMReX/src/driver.cxx at line 85 [3.15972]

B:BD[8.57] → [8.57:161]

    nbx.setSmall(d, md - rd / (1 << (lev + 1)) + 1);
    nbx.setBig(d, md + rd / (1 << (lev + 1)) - 2);

[8.57]

[5.1158]

    nbx.setSmall(d, md - rd / (1 << (level + 1)) + 1);
    nbx.setBig(d, md + rd / (1 << (level + 1)) - 2);

Deletion in AMReX/src/driver.cxx at line 88 [3.15972]
B:BD[5.1162] → [7.253:287]
```
  cout << "lev: " << lev << "\n";
```
Replacement in AMReX/src/driver.cxx at line 89 [3.15972]
B:BD[5.1196] → [7.288:326]
```
  const BoxArray &ba = boxArray(lev);
```
[5.1196]
[5.1196]
```
  const BoxArray &ba = boxArray(level);
```

Insertion in AMReX/src/driver.cxx at line 91 [3.15972]

[5.1244]

  return;
  const int gi = CCTK_GroupIndex("AMReX::regrid_tag");
  assert(gi >= 0);
  const int vi = 0;
  const int tl = 0;
  auto &restrict leveldata = ghext->leveldata.at(level);
  auto &restrict groupdata = leveldata.groupdata.at(gi);
  MultiFab &mfab = *groupdata.mfab.at(tl);
  auto mfitinfo = MFItInfo().SetDynamic(true).EnableTiling({1024000, 16, 32});
#pragma omp parallel
  for (MFIter mfi(mfab, mfitinfo); mfi.isValid(); ++mfi) {
    const Box &fbx = mfi.fabbox();
    const Box &bx = mfi.tilebox();
    const Dim3 imin = lbound(bx);
    int ash[3], lsh[3];
    for (int d = 0; d < dim; ++d)
      ash[d] = fbx[orient(d, 1)] - fbx[orient(d, 0)] + 1;
    for (int d = 0; d < dim; ++d)
      lsh[d] = bx[orient(d, 1)] - bx[orient(d, 0)] + 1;
    const Array4<CCTK_REAL> &ctagarr = groupdata.mfab.at(tl)->array(mfi);
    const CCTK_REAL *restrict ctags = ctagarr.ptr(imin.x, imin.y, imin.z, vi);
    const Array4<char> &atagarr = tags.array(mfi);
    char *restrict atags = atagarr.ptr(imin.x, imin.y, imin.z, vi);
    constexpr int di = 1;
    const int dj = di * ash[0];
    const int dk = dj * ash[1];
    for (int k = 0; k < lsh[2]; ++k) {
      for (int j = 0; j < lsh[1]; ++j) {
#pragma omp simd
        for (int i = 0; i < lsh[0]; ++i) {
          int idx = di * i + dj * j + dk * k;
          atags[idx] = ctags[idx] == 0.0 ? TagBox::CLEAR : TagBox::SET;
        }
      }
    }
  }

Insertion in AMReX/src/driver.cxx at line 136 [3.15972]
[5.1328]
[5.1328]
```
void SetupLevel(int level);
```

Replacement in AMReX/src/driver.cxx at line 230 [3.15972]

B:BD[3.17301] → [5.1900:2028]

∅:D[5.2028] → [9.1143:1234]

B:BD[9.1143] → [9.1143:1234]

  const int numlevels = maxnumlevels;
  ghext->leveldata.resize(numlevels);
  for (int level = 0; level < numlevels; ++level) {
    GHExt::LevelData &leveldata = ghext->leveldata.at(level);
    leveldata.level = level;

[3.17301]

[5.2029]

  // Create coarse grid
  const int level = 0;
  CCTK_REAL time = 0.0; // dummy time
  ghext->amrmesh->MakeNewGrids(time);
  SetupLevel(level);

Replacement in AMReX/src/driver.cxx at line 236 [3.15972]

B:BD[5.2030] → [5.2030:2070]

    CCTK_REAL time = 0.0; // dummy time

[5.2030]

[3.17361]

  // CCTK_VINFO("BoxArray level %d:", level);
  // cout << ghext->amrmesh->boxArray(level) << "\n";
  // CCTK_VINFO("DistributionMap level %d:", level);
  // cout << ghext->amrmesh->DistributionMap(level) << "\n";
  return 0; // unused
} // namespace AMReX
void CreateRefinedGrid(int level) {
  CCTK_VINFO("CreateRefinedGrid level %d", level);

Replacement in AMReX/src/driver.cxx at line 247 [3.15972]

B:BD[3.17362] → [5.2071:2365]

B:BD[5.2365] → [7.358:472]

∅:D[7.472] → [5.2400:2406]

B:BD[5.2400] → [5.2400:2406]

    // Create grid structure
    if (level == 0) {
      // Create coarse grid
      ghext->amrmesh->MakeNewGrids(time);
    } else {
      // Create refined grid
      int new_finest = -999;
      Vector<BoxArray> new_grids;
      ghext->amrmesh->MakeNewGrids(0, time, new_finest, new_grids);
      // cout << "level=" << level << " new_finest=" << new_finest << "\n";
      // assert(new_finest == level);
    }

[3.17362]

[5.2406]

  // Create refined grid
  CCTK_REAL time = 0.0; // dummy time
  int new_finest = -999;
  Vector<BoxArray> new_grids;
  ghext->amrmesh->MakeNewGrids(0, time, new_finest, new_grids);
  cout << "level=" << level << "\n";
  cout << "new_finest=" << new_finest << "\n";
  assert(new_finest == level - 1 || new_finest == level);
  ghext->amrmesh->SetFinestLevel(new_finest);
  ghext->amrMesh->SetBoxArray(level, new_grids.at(level));
  ghext->amrMesh->SetDistributionMap(...);

Replacement in AMReX/src/driver.cxx at line 259 [3.15972]

B:BD[5.2407] → [5.2407:2629]

    // CCTK_VINFO("BoxArray level %d:", level);
    // cout << ghext->amrmesh->boxArray(level) << "\n";
    // CCTK_VINFO("DistributionMap level %d:", level);
    // cout << ghext->amrmesh->DistributionMap(level) << "\n";

[5.2407]

[5.2629]

  if (new_finest == level)
    SetupLevel(level);
}

Replacement in AMReX/src/driver.cxx at line 263 [3.15972]

∅:D[5.2630] → [10.959:1356]

B:BD[3.17362] → [10.959:1356]

    const int numgroups = CCTK_NumGroups();
    leveldata.groupdata.resize(numgroups);
    for (int gi = 0; gi < numgroups; ++gi) {
      cGroup group;
      int ierr = CCTK_GroupData(gi, &group);
      assert(!ierr);
      assert(group.grouptype == CCTK_GF);
      assert(group.vartype == CCTK_VARIABLE_REAL);
      assert(group.disttype == CCTK_DISTRIB_DEFAULT);
      assert(group.dim == dim);

[5.2630]

[11.827]

void SetupLevel(int level) {
  DECLARE_CCTK_PARAMETERS;
  CCTK_VINFO("SetupLevel level %d", level);

Replacement in AMReX/src/driver.cxx at line 267 [3.15972]

B:BD[11.828] → [10.1357:1530]

      GHExt::LevelData::GroupData &groupdata = leveldata.groupdata.at(gi);
      groupdata.firstvarindex = CCTK_FirstVarIndexI(gi);
      groupdata.numvars = group.numvars;

[11.828]

[11.1032]

  assert(level == int(ghext->leveldata.size()));
  ghext->leveldata.resize(level + 1);
  GHExt::LevelData &leveldata = ghext->leveldata.at(level);
  leveldata.level = level;

Replacement in AMReX/src/driver.cxx at line 272 [3.15972]

B:BD[11.1033] → [10.1531:1735]

B:BD[10.1735] → [9.1235:1396]

∅:D[9.1396] → [10.1812:1820]

B:BD[10.1812] → [10.1812:1820]

∅:D[10.1820] → [12.286:292]

B:BD[12.286] → [12.286:292]

      // Allocate grid hierarchies
      groupdata.mfab.resize(group.numtimelevels);
      for (int tl = 0; tl < int(groupdata.mfab.size()); ++tl) {
        groupdata.mfab.at(tl) = make_unique<MultiFab>(
            ghext->amrmesh->boxArray(leveldata.level),
            ghext->amrmesh->DistributionMap(leveldata.level), groupdata.numvars,
            ghost_size);
      }
    }

[11.1033]

[10.1821]

  const int numgroups = CCTK_NumGroups();
  leveldata.groupdata.resize(numgroups);
  for (int gi = 0; gi < numgroups; ++gi) {
    cGroup group;
    int ierr = CCTK_GroupData(gi, &group);
    assert(!ierr);
    assert(group.grouptype == CCTK_GF);
    assert(group.vartype == CCTK_VARIABLE_REAL);
    assert(group.disttype == CCTK_DISTRIB_DEFAULT);
    assert(group.dim == dim);

Replacement in AMReX/src/driver.cxx at line 283 [3.15972]

B:BD[10.1822] → [10.1822:1839]

  } // for level

[10.1822]

[13.415]

    GHExt::LevelData::GroupData &groupdata = leveldata.groupdata.at(gi);
    groupdata.firstvarindex = CCTK_FirstVarIndexI(gi);
    groupdata.numvars = group.numvars;

Replacement in AMReX/src/driver.cxx at line 287 [3.15972]

∅:D[13.416] → [6.1197:1219]

B:BD[3.17460] → [6.1197:1219]

B:BD[6.1219] → [5.2631:2652]

  return 0; // unused
} // namespace AMReX

[13.416]

[6.1221]

    // Allocate grid hierarchies
    groupdata.mfab.resize(group.numtimelevels);
    for (int tl = 0; tl < int(groupdata.mfab.size()); ++tl) {
      groupdata.mfab.at(tl) = make_unique<MultiFab>(
          ghext->amrmesh->boxArray(leveldata.level),
          ghext->amrmesh->DistributionMap(leveldata.level), groupdata.numvars,
          ghost_size);
    }
  }
}

Insertion in AMReX/src/driver.hxx at line 73 [3.15275]
[3.15908]
[14.1361]
```
void CreateRefinedGrid(int level);
```

Insertion in AMReX/src/schedule.cxx at line 40 [13.531]

[13.1182]

[15.4208]

// During initialization, the schedule should traverse only a single
// level. At all other times, all levels should be traversed.
int current_level = -1;

Replacement in AMReX/src/schedule.cxx at line 313 [13.531]
B:BD[11.2503] → [11.2503:2545]
```
  CCTK_Traverse(cctkGH, "CCTK_BASEGRID");
```
[11.2503]
[11.2545]
```
  if (config->recovered) {
    // Recover
```

Replacement in AMReX/src/schedule.cxx at line 316 [13.531]

B:BD[11.2546] → [11.2546:2964]

  const char *recovery_mode = *static_cast<const char *const *>(
      CCTK_ParameterGet("recovery_mode", "Cactus", nullptr));
  if (!config->recovered || !CCTK_Equals(recovery_mode, "strict")) {
    // Set up initial conditions
    CCTK_Traverse(cctkGH, "CCTK_INITIAL");
    CCTK_Traverse(cctkGH, "CCTK_POSTINITIAL");
    CCTK_Traverse(cctkGH, "CCTK_POSTPOSTINITIAL");
    CCTK_Traverse(cctkGH, "CCTK_POSTSTEP");
  }

[11.2546]

[11.2964]

    const char *recovery_mode = *static_cast<const char *const *>(
        CCTK_ParameterGet("recovery_mode", "Cactus", nullptr));
    CCTK_Traverse(cctkGH, "CCTK_BASEGRID");

Replacement in AMReX/src/schedule.cxx at line 321 [13.531]

B:BD[11.2965] → [11.2965:2992]

  if (config->recovered) {

[11.2965]

[11.2992]

    if (!CCTK_Equals(recovery_mode, "strict")) {
      // Set up initial conditions
      CCTK_Traverse(cctkGH, "CCTK_INITIAL");
      CCTK_Traverse(cctkGH, "CCTK_POSTINITIAL");
      CCTK_Traverse(cctkGH, "CCTK_POSTPOSTINITIAL");
    }

Insertion in AMReX/src/schedule.cxx at line 331 [13.531]

[11.3118]


  } else {
    // Set up initial conditions
    for (;;) {
      current_level = ghext->amrmesh->finestLevel();
      CCTK_Traverse(cctkGH, "CCTK_BASEGRID");
      CCTK_Traverse(cctkGH, "CCTK_INITIAL");
      CCTK_Traverse(cctkGH, "CCTK_POSTINITIAL");
      CCTK_Traverse(cctkGH, "CCTK_POSTPOSTINITIAL");
      const int old_numlevels = ghext->amrmesh->finestLevel() + 1;
      CreateRefinedGrid(current_level + 1);
      const int new_numlevels = ghext->amrmesh->finestLevel() + 1;
      assert(new_numlevels == old_numlevels ||
             new_numlevels == old_numlevels + 1);
      cout << "old_numlevels=" << old_numlevels << "\n";
      cout << "new_numlevels=" << new_numlevels << "\n";
      cout << "max_numlevels=" << int(ghext->amrmesh->maxLevel() + 1) << "\n";
      // Did we create a new level?
      if (new_numlevels == old_numlevels)
        break;
      // Did we create all possible levels?
      if (new_numlevels == int(ghext->amrmesh->maxLevel() + 1))
        break;
    }
    current_level = -1;

Insertion in AMReX/src/schedule.cxx at line 362 [13.531]
[11.3157]
[11.3157]
```
  CCTK_Traverse(cctkGH, "CCTK_POSTSTEP");
```

Replacement in AMReX/src/schedule.cxx at line 489 [13.531]

B:BD[16.2229] → [10.2770:2828]

      for (auto &restrict leveldata : ghext->leveldata) {

[16.2229]

[10.2828]


      auto callfunc = [&](auto &restrict leveldata) {

Insertion in AMReX/src/schedule.cxx at line 501 [13.531]

[10.3097]

[16.2391]

      };
      if (current_level < 0) {
        // Loop over all levels
        // TODO: parallelize this loop
        for (auto &restrict leveldata : ghext->leveldata)
          callfunc(leveldata);
      } else {
        // Loop over a single level
        auto &restrict leveldata = ghext->leveldata.at(current_level);
        callfunc(leveldata);

Insertion in WaveToyAMReX/interface.ccl at line 4 [3.10196]
[3.10269]
[14.1523]
```
INHERITS: AMReX
```

Insertion in WaveToyAMReX/schedule.ccl at line 9 [3.9594]

[14.1890]

[3.9833]

SCHEDULE WaveToyAMReX_Tag AT postinitial
{
  LANG: C
} "Tag to-be-refined regions"

Insertion in WaveToyAMReX/src/wavetoy.cxx at line 56 [3.37]

[3.5451]


extern "C" void WaveToyAMReX_Tag(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;

Insertion in WaveToyAMReX/src/wavetoy.cxx at line 61 [3.37]

[3.5452]

  CCTK_REAL x0[dim], dx[dim];
  for (int d = 0; d < dim; ++d) {
    x0[d] = CCTK_ORIGIN_SPACE(d);
    dx[d] = CCTK_DELTA_SPACE(d);
  }
  for (int k = 0; k < cctk_lsh[2]; ++k) {
    for (int j = 0; j < cctk_lsh[1]; ++j) {
#pragma omp simd
      for (int i = 0; i < cctk_lsh[0]; ++i) {
        const int idx = CCTK_GFINDEX3D(cctkGH, i, j, k);
        CCTK_REAL x = x0[0] + (cctk_lbnd[0] + i) * dx[0];
        CCTK_REAL y = x0[1] + (cctk_lbnd[1] + j) * dx[1];
        CCTK_REAL z = x0[2] + (cctk_lbnd[2] + k) * dx[2];
        CCTK_REAL r = sqrt(sqr(x) + sqr(y) + sqr(z));
        tag[idx] = r <= 0.5;
      }
    }
  }
}

Replacement in azure-pipelines/amrex.par at line 12 [17.12]
B:BD[5.6784] → [2.0:26]
```
Cactus::cctk_itlast = 128
```
[5.6784]
[8.1850]
```
Cactus::cctk_itlast = 0   #TODO 128
```