eschnett/cactusamrex - Change BJDGFYBMECTJG7BHLNHLSCUCBVYHAY6OGY37FIJP6JDGNDXQNQVAC

Disable ghost zones for fluxes

Created by Erik Schnetter on October 18, 2019

BJDGFYBMECTJG7BHLNHLSCUCBVYHAY6OGY37FIJP6JDGNDXQNQVAC

Dependencies

In channels

main

master

Change contents

Replacement in AMReX/src/driver.cxx at line 89 [13.15972]
∅:D[11.121] → [12.1473:1511]
B:BD[14.1190] → [12.1473:1511]
```
    GridPtrDesc grid(leveldata, mfi);
```
[11.121]
[14.1501]
```
    GridPtrDesc1 grid(leveldata, groupdata, mfi);
```

Replacement in AMReX/src/driver.cxx at line 91 [13.15972]

B:BD[14.1502] → [15.1501:1572]

B:BD[15.1572] → [12.1512:1568]

    const Array4<CCTK_REAL> &vars = groupdata.mfab.at(tl)->array(mfi);
    CCTK_REAL *restrict const err = grid.ptr(vars, vi);

[14.1502]

[16.0]

    const Array4<const CCTK_REAL> &err_array4 =
        groupdata.mfab.at(tl)->array(mfi);
    const GF3D1<const CCTK_REAL> err_ = grid.gf3d(err_array4, vi);

Replacement in AMReX/src/driver.cxx at line 96 [13.15972]

B:BD[15.1698] → [11.122:193]

B:BD[11.193] → [16.56:178]

∅:D[16.178] → [11.299:377]

B:BD[11.299] → [11.299:377]

B:BD[11.377] → [17.83:91]

    grid.loop_int(groupdata.indextype, [&](const Loop::PointDesc &p) {
      tags_array4(grid.cactus_offset.x + p.i, grid.cactus_offset.y + p.j,
                  grid.cactus_offset.z + p.k) =
          err[p.idx] >= regrid_error_threshold ? TagBox::SET : TagBox::CLEAR;
    });

[15.1698]

[6.42]

    grid.loop_idx(
        where_t::interior, groupdata.indextype, [&](const Loop::PointDesc &p) {
          tags_array4(grid.cactus_offset.x + p.i, grid.cactus_offset.y + p.j,
                      grid.cactus_offset.z + p.k) =
              err_(p.I) >= regrid_error_threshold ? TagBox::SET : TagBox::CLEAR;
        });

Replacement in AMReX/src/driver.cxx at line 105 [13.15972]

B:BD[6.212] → [6.212:440]

    // grid.loop_bnd(groupdata.indextype, [&](const Loop::PointDesc &p) {
    //   tags_array4(grid.cactus_offset.x + p.i, grid.cactus_offset.y + p.j,
    //               grid.cactus_offset.z + p.k) = TagBox::CLEAR;
    // });

[6.212]

[11.378]

    if (false) {
      grid.loop_idx(where_t::boundary, groupdata.indextype,
                    [&](const Loop::PointDesc &p) {
                      tags_array4(grid.cactus_offset.x + p.i,
                                  grid.cactus_offset.y + p.j,
                                  grid.cactus_offset.z + p.k) = TagBox::CLEAR;
                    });
    }

Insertion in AMReX/src/driver.cxx at line 184 [13.15972]

[5.1474]

[18.1302]

array<int, dim> get_group_nghostzones(const int gi) {
  DECLARE_CCTK_PARAMETERS;
  assert(gi >= 0);
  const int tags = CCTK_GroupTagsTableI(gi);
  assert(tags >= 0);
  array<CCTK_INT, dim> nghostzones;
  int iret =
      Util_TableGetIntArray(tags, dim, nghostzones.data(), "nghostzones");
  if (iret == UTIL_ERROR_TABLE_NO_SUCH_KEY) {
    // default: use driver parameter
    nghostzones = {ghost_size, ghost_size, ghost_size};
  } else if (iret >= 0) {
    assert(iret == dim);
  } else {
    assert(0);
  }
  return nghostzones;
}

Insertion in AMReX/src/driver.cxx at line 233 [13.15972]
[11.1151]
[18.2191]
```
    groupdata.nghostzones = get_group_nghostzones(gi);
```

Replacement in AMReX/src/driver.cxx at line 244 [13.15972]

∅:D[19.1164] → [18.2335:2365]

∅:D[12.1917] → [18.2335:2365]

B:BD[18.2335] → [18.2335:2365]

B:BD[18.2365] → [11.1436:1509]

      groupdata.mfab.at(tl) =
          make_unique<MultiFab>(gba, dm, groupdata.numvars, ghost_size);

[12.1917]

[20.312]

      groupdata.mfab.at(tl) = make_unique<MultiFab>(
          gba, dm, groupdata.numvars, IntVect(groupdata.nghostzones));

Replacement in AMReX/src/driver.cxx at line 567 [13.15972]

B:BD[11.2982] → [21.285:366]

      auto mfab = make_unique<MultiFab>(gba, dm, groupdata.numvars, ghost_size);

[11.2982]

[20.1377]

      auto mfab = make_unique<MultiFab>(gba, dm, groupdata.numvars,
                                        IntVect(groupdata.nghostzones));

Replacement in AMReX/src/driver.cxx at line 576 [13.15972]

B:BD[22.1105] → [22.1105:1149]

          GridPtrDesc grid(leveldata, mfi);

[22.1105]

[22.1149]

          GridPtrDesc1 grid(leveldata, groupdata, mfi);

Replacement in AMReX/src/driver.cxx at line 579 [13.15972]

B:BD[22.1268] → [22.1268:1465]

            CCTK_REAL *restrict const ptr = grid.ptr(vars, vi);
            grid.loop_all(groupdata.indextype, [&](const Loop::PointDesc &p) {
              ptr[p.idx] = 0.0 / 0.0;
            });

[22.1268]

[22.1465]

            const GF3D1<CCTK_REAL> ptr_ = grid.gf3d(vars, vi);
            grid.loop_idx(
                where_t::everywhere, groupdata.indextype, groupdata.nghostzones,
                [&](const Loop::PointDesc &p) { ptr_(p.I) = 0.0 / 0.0; });

Replacement in AMReX/src/driver.cxx at line 680 [13.15972]

B:BD[23.389] → [9.0:91]

  string banner =
      "AMR driver provided by CarpetX, using AMReX " + amrex::Version();

[23.389]

[23.390]

  string banner = "AMR driver provided by CarpetX, using AMReX " +
                  amrex::Version() +
                  " ("
#ifdef AMREX_USE_MPI
                  "MPI, "
#else
                  "no MPI, "
#endif
#ifdef AMREX_USE_OMP
                  "OpenMP, "
#else
                  "no OpenMP, "
#endif
#ifdef AMREX_USE_GPU
                  "Accelerators, "
#else
                  "no Accelerators, "
#endif
#ifdef AMREX_USE_ASSERTION
                  "DEBUG, "
#else
                  "OPTIMIZED, "
#endif
                  ")";

Insertion in AMReX/src/driver.hxx at line 108 [13.15275]
[16.212]
[5.2742]
```
      array<int, dim> nghostzones;
```

Replacement in AMReX/src/io.cxx at line 154 [24.107]

∅:D[11.7114] → [12.5284:5324]

B:BD[15.6220] → [12.5284:5324]

∅:D[12.5324] → [15.7196:7326]

B:BD[15.7196] → [15.7196:7326]

      GridPtrDesc grid(leveldata, mfi);
      const Array4<CCTK_REAL> &vars = groupdata.mfab.at(tl)->array(mfi);
      vector<const CCTK_REAL *> ptrs(groupdata.numvars);

[11.7114]

[15.7326]

      GridPtrDesc1 grid(leveldata, groupdata, mfi);
      const Array4<const CCTK_REAL> &vars = groupdata.mfab.at(tl)->array(mfi);
      vector<GF3D1<const CCTK_REAL> > ptrs_;
      ptrs_.reserve(groupdata.numvars);

Replacement in AMReX/src/io.cxx at line 159 [24.107]

B:BD[15.7379] → [12.5325:5367]

        ptrs.at(vi) = grid.ptr(vars, vi);

[15.7379]

[11.7115]

        ptrs_.push_back(grid.gf3d(vars, vi));

Replacement in AMReX/src/io.cxx at line 161 [24.107]

B:BD[11.7194] → [11.7194:7798]

      Loop::loop_all(
          cctkGH, groupdata.indextype, [&](const Loop::PointDesc &p) {
            file << cctkGH->cctk_iteration << "\t" << cctkGH->cctk_time << "\t"
                 << leveldata.level << "\t"
                 << mfi.index()
                 // << "\t" << isghost
                 << "\t" << (grid.lbnd[0] + p.i) << "\t" << (grid.lbnd[1] + p.j)
                 << "\t" << (grid.lbnd[2] + p.k) << "\t" << p.x << "\t" << p.y
                 << "\t" << p.z;
            for (const auto &ptr : ptrs)
              file << "\t" << ptr[p.idx];
            file << "\n";
          });

[11.7194]

[11.7798]

      Loop::loop_idx(cctkGH, where_t::everywhere, groupdata.indextype,
                     [&](const Loop::PointDesc &p) {
                       file << cctkGH->cctk_iteration << "\t"
                            << cctkGH->cctk_time << "\t" << leveldata.level
                            << "\t"
                            << mfi.index()
                            // << "\t" << isghost
                            << "\t" << (grid.lbnd[0] + p.i) << "\t"
                            << (grid.lbnd[1] + p.j) << "\t"
                            << (grid.lbnd[2] + p.k) << "\t" << p.x << "\t"
                            << p.y << "\t" << p.z;
                       for (const auto &ptr_ : ptrs_)
                         file << "\t" << ptr_(p.I);
                       file << "\n";
                     });

Insertion in AMReX/src/io.cxx at line 180 [24.107]
[15.7573]
[15.7573]

Replacement in AMReX/src/level.cxx at line 16 [25.4123]

B:BD[25.4240] → [3.22:124]

  Loop::loop_all<1, 1, 1>(
      cctkGH, [&](const Loop::PointDesc &p) { regrid_error[p.idx] = 0; });

[25.4240]

[20.3200]

  const Loop::GF3D<CCTK_REAL, 1, 1, 1> regrid_error_(cctkGH, regrid_error);
  Loop::loop<1, 1, 1>(
      cctkGH, where_t::everywhere,
      [&](const Loop::PointDesc &p) { regrid_error_(p.I) = 0; });

Insertion in AMReX/src/level.cxx at line 27 [25.4123]

[25.4343]

  const Loop::GF3D<CCTK_REAL, 1, 1, 1> refinement_level_(cctkGH,
                                                         refinement_level);

Replacement in AMReX/src/level.cxx at line 37 [25.4123]

B:BD[25.4449] → [3.125:233]

  Loop::loop_all<1, 1, 1>(
      cctkGH, [&](const Loop::PointDesc &p) { refinement_level[p.idx] = lev; });

[25.4449]

[25.4710]

  Loop::loop<1, 1, 1>(
      cctkGH, where_t::everywhere,
      [&](const Loop::PointDesc &p) { refinement_level_(p.I) = lev; });

Insertion in AMReX/src/loop.hxx at line 105 [12.5481]
[4.1555]
[11.8940]
```
// TODO: remove this
```

Insertion in AMReX/src/loop.hxx at line 139 [12.5481]

[11.9869]

template <typename T> struct GF3D1 {
  typedef T value_type;
  T *restrict ptr;
#ifdef CCTK_DEBUG
  array<int, dim> imin, imax;
  array<int, dim> ash;
#endif
  static constexpr int di = 1;
  int dj, dk;
  int off;
  inline GF3D1(T *restrict ptr, const array<int, dim> &imin,
               const array<int, dim> &imax, const array<int, dim> &ash)
      : ptr(ptr),
#ifdef CCTK_DEBUG
        imin(imin), imax(imax), ash(ash),
#endif
        dj(di * ash[0]), dk(dj * ash[1]),
        off(imin[0] * di + imin[1] * dj + imin[2] * dk) {
  }
  inline GF3D1(const cGH *restrict cctkGH, const array<int, dim> &indextype,
               const array<int, dim> &nghostzones, T *restrict ptr) {
    for (int d = 0; d < dim; ++d)
      assert(indextype[d] == 0 || indextype[d] == 1);
    for (int d = 0; d < dim; ++d) {
      assert(nghostzones[d] >= 0);
      assert(nghostzones[d] <= cctkGH->cctk_nghostzones[d]);
    }
    array<int, dim> imin, imax;
    for (int d = 0; d < dim; ++d) {
      imin[d] = cctkGH->cctk_nghostzones[d] - nghostzones[d];
      imax[d] = cctkGH->cctk_lsh[d] + (1 - indextype[d]) -
                (cctkGH->cctk_nghostzones[d] - nghostzones[d]);
    }
    array<int, dim> ash;
    for (int d = 0; d < dim; ++d)
      ash[d] = cctkGH->cctk_ash[d] + (1 - indextype[d]) -
               2 * (cctkGH->cctk_nghostzones[d] - nghostzones[d]);
    *this = GF3D1(ptr, imin, imax, ash);
  }
  inline int offset(int i, int j, int k) const {
    // These index checks prevent vectorization. We thus only enable
    // them in debug mode.
#ifdef CCTK_DEBUG
    assert(i >= imin[0] && i < imax[0]);
    assert(j >= imin[1] && j < imax[1]);
    assert(k >= imin[2] && k < imax[2]);
#endif
    return i * di + j * dj + k * dk - off;
  }
  inline T &restrict operator()(int i, int j, int k) const {
    return ptr[offset(i, j, k)];
  }
  inline T &restrict operator()(const vect<int, dim> &I) const {
    return ptr[offset(I[0], I[1], I[2])];
  }
};

Insertion in AMReX/src/loop.hxx at line 198 [12.5481]
[11.9951]
[11.9951]
```
enum class where_t { everywhere, interior, boundary };
```

Replacement in AMReX/src/loop.hxx at line 298 [12.5481]

B:BD[11.10799] → [11.10799:10835]

  void loop_all(const F &f) const {

[11.10799]

[11.10835]

  void loop_all(const array<int, dim> &group_nghostzones, const F &f) const {

Replacement in AMReX/src/loop.hxx at line 302 [12.5481]

B:BD[12.7505] → [12.7505:7538]

      imin[d] = max(tmin[d], 0);

[12.7505]

[11.10889]

      int ghost_offset = nghostzones[d] - group_nghostzones[d];
      imin[d] = max(tmin[d], ghost_offset);

Replacement in AMReX/src/loop.hxx at line 305 [12.5481]

B:BD[11.10956] → [11.10956:10997]

                    lsh[d] + offset[d]);

[11.10956]

[12.7576]

                    lsh[d] + offset[d] - ghost_offset);

Replacement in AMReX/src/loop.hxx at line 313 [12.5481]

B:BD[11.11088] → [11.11088:11124]

  void loop_int(const F &f) const {

[11.11088]

[11.11124]

  void loop_int(const array<int, dim> &group_nghostzones, const F &f) const {

Replacement in AMReX/src/loop.hxx at line 328 [12.5481]

B:BD[11.11394] → [11.11394:11430]

  void loop_bnd(const F &f) const {

[11.11394]

[11.11430]

  void loop_bnd(const array<int, dim> &group_nghostzones, const F &f) const {

Replacement in AMReX/src/loop.hxx at line 338 [12.5481]

B:BD[12.8385] → [12.8385:8410]

B:BD[12.8410] → [11.11485:11527]

            imin[d] = 0;
            imax[d] = lsh[d] + offset[d];

[12.8385]

[12.8440]

            int ghost_offset = nghostzones[d] - group_nghostzones[d];
            imin[d] = ghost_offset;
            imax[d] = lsh[d] + offset[d] - ghost_offset;

Replacement in AMReX/src/loop.hxx at line 368 [12.5481]

B:BD[11.11836] → [11.11836:13011]

  template <typename F>
  void loop_all(const array<int, dim> &indextype, const F &f) const {
    // typedef void (GridDescBase::*funptr)(const F &f) const;
    // constexpr array<funptr, 8> funptrs{
    //     &GridDescBase::loop_all<0, 0, 0, F>,
    //     &GridDescBase::loop_all<1, 0, 0, F>,
    //     &GridDescBase::loop_all<0, 1, 0, F>,
    //     &GridDescBase::loop_all<1, 1, 0, F>,
    //     &GridDescBase::loop_all<0, 0, 1, F>,
    //     &GridDescBase::loop_all<1, 0, 1, F>,
    //     &GridDescBase::loop_all<0, 1, 1, F>,
    //     &GridDescBase::loop_all<1, 1, 1, F>,
    // };
    // return (this->*funptrs[indextype[0] + 2 * indextype[1] + 4 *
    // indextype[2]])(
    //     f);
    switch (indextype[0] + 2 * indextype[1] + 4 * indextype[2]) {
    case 0b000:
      return loop_all<0, 0, 0>(f);
    case 0b001:
      return loop_all<1, 0, 0>(f);
    case 0b010:
      return loop_all<0, 1, 0>(f);
    case 0b011:
      return loop_all<1, 1, 0>(f);
    case 0b100:
      return loop_all<0, 0, 1>(f);
    case 0b101:
      return loop_all<1, 0, 1>(f);
    case 0b110:
      return loop_all<0, 1, 1>(f);
    case 0b111:
      return loop_all<1, 1, 1>(f);

[11.11836]

[11.13011]

  template <int CI, int CJ, int CK, typename F>
  void loop(where_t where, const array<int, dim> &group_nghostzones,
            const F &f) const {
    switch (where) {
    case where_t::everywhere:
      return loop_all<CI, CJ, CK>(group_nghostzones, f);
    case where_t::interior:
      return loop_int<CI, CJ, CK>(group_nghostzones, f);
    case where_t::boundary:
      return loop_bnd<CI, CJ, CK>(group_nghostzones, f);

Insertion in AMReX/src/loop.hxx at line 383 [12.5481]

[11.13052]

  template <int CI, int CJ, int CK, typename F>
  void loop(where_t where, const F &f) const {
    loop<CI, CJ, CK>(where, nghostzones, f);
  }

Replacement in AMReX/src/loop.hxx at line 389 [12.5481]

B:BD[11.13076] → [11.13076:13146]

  void loop_int(const array<int, dim> &indextype, const F &f) const {

[11.13076]

[11.13146]

  void loop_idx(where_t where, const array<int, dim> &indextype,
                const array<int, dim> &group_nghostzones, const F &f) const {

Replacement in AMReX/src/loop.hxx at line 393 [12.5481]
B:BD[11.13228] → [11.13228:13263]
```
      return loop_int<0, 0, 0>(f);
```
[11.13228]
[11.13263]
```
      return loop<0, 0, 0>(where, group_nghostzones, f);
```
Replacement in AMReX/src/loop.hxx at line 395 [12.5481]
B:BD[11.13279] → [11.13279:13314]
```
      return loop_int<1, 0, 0>(f);
```
[11.13279]
[11.13314]
```
      return loop<1, 0, 0>(where, group_nghostzones, f);
```
Replacement in AMReX/src/loop.hxx at line 397 [12.5481]
B:BD[11.13330] → [11.13330:13365]
```
      return loop_int<0, 1, 0>(f);
```
[11.13330]
[11.13365]
```
      return loop<0, 1, 0>(where, group_nghostzones, f);
```
Replacement in AMReX/src/loop.hxx at line 399 [12.5481]
B:BD[11.13381] → [11.13381:13416]
```
      return loop_int<1, 1, 0>(f);
```
[11.13381]
[11.13416]
```
      return loop<1, 1, 0>(where, group_nghostzones, f);
```
Replacement in AMReX/src/loop.hxx at line 401 [12.5481]
B:BD[11.13432] → [11.13432:13467]
```
      return loop_int<0, 0, 1>(f);
```
[11.13432]
[11.13467]
```
      return loop<0, 0, 1>(where, group_nghostzones, f);
```
Replacement in AMReX/src/loop.hxx at line 403 [12.5481]
B:BD[11.13483] → [11.13483:13518]
```
      return loop_int<1, 0, 1>(f);
```
[11.13483]
[11.13518]
```
      return loop<1, 0, 1>(where, group_nghostzones, f);
```
Replacement in AMReX/src/loop.hxx at line 405 [12.5481]
B:BD[11.13534] → [11.13534:13569]
```
      return loop_int<0, 1, 1>(f);
```
[11.13534]
[11.13569]
```
      return loop<0, 1, 1>(where, group_nghostzones, f);
```
Replacement in AMReX/src/loop.hxx at line 407 [12.5481]
B:BD[11.13585] → [11.13585:13620]
```
      return loop_int<1, 1, 1>(f);
```
[11.13585]
[11.13620]
```
      return loop<1, 1, 1>(where, group_nghostzones, f);
```

Replacement in AMReX/src/loop.hxx at line 414 [12.5481]

B:BD[11.13685] → [11.13685:14259]

∅:D[11.14259] → [12.9129:9135]

B:BD[12.9129] → [12.9129:9135]

  void loop_bnd(const array<int, dim> &indextype, const F &f) const {
    switch (indextype[0] + 2 * indextype[1] + 4 * indextype[2]) {
    case 0b000:
      return loop_bnd<0, 0, 0>(f);
    case 0b001:
      return loop_bnd<1, 0, 0>(f);
    case 0b010:
      return loop_bnd<0, 1, 0>(f);
    case 0b011:
      return loop_bnd<1, 1, 0>(f);
    case 0b100:
      return loop_bnd<0, 0, 1>(f);
    case 0b101:
      return loop_bnd<1, 0, 1>(f);
    case 0b110:
      return loop_bnd<0, 1, 1>(f);
    case 0b111:
      return loop_bnd<1, 1, 1>(f);
    default:
      assert(0);
    }

[11.13685]

[12.9135]

  void loop_idx(where_t where, const array<int, dim> &indextype,
                const F &f) const {
    loop_idx(where, indextype, nghostzones, f);

Replacement in AMReX/src/loop.hxx at line 420 [12.5481]

B:BD[12.9143] → [11.14260:14401]

template <int CI, int CJ, int CK, typename F>
void loop_all(const cGH *cctkGH, const F &f) {
  GridDescBase(cctkGH).loop_all<CI, CJ, CK>(f);

[12.9143]

[11.14401]

template <typename F>
void loop_idx(const cGH *cctkGH, where_t where,
              const array<int, dim> &indextype,
              const array<int, dim> &nghostzones, const F &f) {
  GridDescBase(cctkGH).loop_idx(where, indextype, nghostzones, f);

Replacement in AMReX/src/loop.hxx at line 427 [12.5481]

B:BD[11.14404] → [11.14404:14545]

template <int CI, int CJ, int CK, typename F>
void loop_int(const cGH *cctkGH, const F &f) {
  GridDescBase(cctkGH).loop_int<CI, CJ, CK>(f);

[11.14404]

[12.9248]

template <typename F>
void loop_idx(const cGH *cctkGH, where_t where,
              const array<int, dim> &indextype, const F &f) {
  GridDescBase(cctkGH).loop_idx(where, indextype, f);

Replacement in AMReX/src/loop.hxx at line 434 [12.5481]

B:BD[11.14592] → [11.14592:14687]

void loop_bnd(const cGH *cctkGH, const F &f) {
  GridDescBase(cctkGH).loop_bnd<CI, CJ, CK>(f);

[11.14592]

[12.9356]

void loop(const cGH *cctkGH, where_t where, const F &f) {
  GridDescBase(cctkGH).loop<CI, CJ, CK>(where, f);

Replacement in AMReX/src/loop.hxx at line 438 [12.5481]

B:BD[12.9359] → [11.14688:14838]

template <typename F>
void loop_all(const cGH *cctkGH, const array<int, dim> &indextype, const F &f) {
  GridDescBase(cctkGH).loop_all(indextype, f);

[12.9359]

[12.9464]

// Keep these for convenience
template <int CI, int CJ, int CK, typename F>
void loop_all(const cGH *cctkGH, const F &f) {
  loop<CI, CJ, CK>(cctkGH, where_t::everywhere, f);

Replacement in AMReX/src/loop.hxx at line 444 [12.5481]

B:BD[12.9467] → [11.14839:14989]

template <typename F>
void loop_int(const cGH *cctkGH, const array<int, dim> &indextype, const F &f) {
  GridDescBase(cctkGH).loop_int(indextype, f);

[12.9467]

[11.14989]

template <int CI, int CJ, int CK, typename F>
void loop_int(const cGH *cctkGH, const F &f) {
  loop<CI, CJ, CK>(cctkGH, where_t::interior, f);

Replacement in AMReX/src/loop.hxx at line 449 [12.5481]

B:BD[11.14992] → [11.14992:15142]

template <typename F>
void loop_bnd(const cGH *cctkGH, const array<int, dim> &indextype, const F &f) {
  GridDescBase(cctkGH).loop_bnd(indextype, f);

[11.14992]

[11.15142]

template <int CI, int CJ, int CK, typename F>
void loop_bnd(const cGH *cctkGH, const F &f) {
  loop<CI, CJ, CK>(cctkGH, where_t::boundary, f);

Replacement in AMReX/src/reduction.cxx at line 51 [26.870]

B:BD[11.15266] → [16.3184:3258]

                          const GridPtrDesc &grid, const T *restrict ptr,

[11.15266]

[16.3258]

                          const GridPtrDesc1 &grid, const GF3D1<const T> ptr_,

Replacement in AMReX/src/reduction.cxx at line 59 [26.870]

B:BD[26.1355] → [11.15340:15409]

B:BD[11.15409] → [16.3338:3628]

∅:D[16.3628] → [12.9750:9756]

∅:D[11.15468] → [12.9750:9756]

B:BD[12.9750] → [12.9750:9756]

  grid.loop_int(groupdata.indextype, [&](const Loop::PointDesc &p) {
    if (!finemask_array4 || !(*finemask_array4)(grid.cactus_offset.x + p.i,
                                                grid.cactus_offset.y + p.j,
                                                grid.cactus_offset.z + p.k))
      red = reduction<T>(red, reduction<T>(dV, ptr[p.idx]));
  });

[26.1355]

[12.9756]

  grid.loop_idx(
      where_t::interior, groupdata.indextype, [&](const Loop::PointDesc &p) {
        if (!finemask_array4 || !(*finemask_array4)(grid.cactus_offset.x + p.i,
                                                    grid.cactus_offset.y + p.j,
                                                    grid.cactus_offset.z + p.k))
          red = reduction<T>(red, reduction<T>(dV, ptr_(p.I)));
      });

Replacement in AMReX/src/reduction.cxx at line 80 [26.870]

B:BD[26.2242] → [20.3204:3256]

    if (!groupdata.valid.at(tl).at(vi).valid_int) {

[26.2242]

[20.3256]

    if (!groupdata.valid.at(tl).at(vi).valid_int)

Deletion in AMReX/src/reduction.cxx at line 85 [26.870]
B:BD[20.3467] → [20.3467:3473]
```
    }
```
Replacement in AMReX/src/reduction.cxx at line 101 [26.870]
B:BD[16.4619] → [16.4619:4659]
```
      GridPtrDesc grid(leveldata, mfi);
```
[16.4619]
[16.4659]
```
      GridPtrDesc1 grid(leveldata, groupdata, mfi);
```

Replacement in AMReX/src/reduction.cxx at line 106 [26.870]

B:BD[16.4875] → [16.4875:5011]

      const CCTK_REAL *restrict ptr = grid.ptr(vars, vi);
      red += reduce_array(geom, groupdata, grid, ptr, finemask_array4.get());

[16.4875]

[26.2713]

      const GF3D1<const CCTK_REAL> ptr_ = grid.gf3d(vars, vi);
      red += reduce_array(geom, groupdata, grid, ptr_, finemask_array4.get());

Deletion in AMReX/src/schedule.cxx at line 49 [27.531]
B:BD[27.1100] → [26.4174:4207]
```
vector<cGH> thread_local_cctkGH;
```

Replacement in AMReX/src/schedule.cxx at line 54 [27.531]

B:BD[12.10714] → [12.10714:10752]

vector<TileBox> thread_local_tilebox;

[12.10714]

[20.3510]


struct thread_local_info_t {
  // TODO: store only MFIter here; recalculate other things from it
  cGH cctkGH;
  TileBox tilebox;
  MFIter *mfiter;
  unsigned char padding[128]; // Prevent false sharing
};
vector<unique_ptr<thread_local_info_t> > thread_local_info;
vector<unique_ptr<thread_local_info_t> > saved_thread_local_info;

Replacement in AMReX/src/schedule.cxx at line 110 [27.531]

B:BD[11.15689] → [11.15689:15768]

    const cGH *restrict threadGH = &AMReX::thread_local_cctkGH.at(thread_num);

[11.15689]

[11.15768]

    const cGH *restrict threadGH =
        &AMReX::thread_local_info.at(thread_num)->cctkGH;

Replacement in AMReX/src/schedule.cxx at line 117 [27.531]

B:BD[12.11749] → [12.11749:11799]

      AMReX::thread_local_tilebox.at(thread_num);

[12.11749]

[12.11799]

      AMReX::thread_local_info.at(thread_num)->tilebox;

Insertion in AMReX/src/schedule.cxx at line 214 [27.531]

[12.13967]

    : GridDesc(leveldata, mfi) {
  const Box &fbx = mfi.fabbox(); // allocated array
  cactus_offset = lbound(fbx);
}
GridPtrDesc1::GridPtrDesc1(const GHExt::LevelData &leveldata,
                           const GHExt::LevelData::GroupData &groupdata,
                           const MFIter &mfi)

Insertion in AMReX/src/schedule.cxx at line 225 [27.531]

[12.14083]

  for (int d = 0; d < dim; ++d) {
    assert(groupdata.nghostzones[d] >= 0);
    assert(groupdata.nghostzones[d] <= nghostzones[d]);
  }
  for (int d = 0; d < dim; ++d)
    gimin[d] = nghostzones[d] - groupdata.nghostzones[d];
  for (int d = 0; d < dim; ++d)
    gimax[d] = lsh[d] + (1 - groupdata.indextype[d]) -
               (nghostzones[d] - groupdata.nghostzones[d]);
  for (int d = 0; d < dim; ++d)
    gash[d] = ash[d] + (1 - groupdata.indextype[d]) -
              2 * (nghostzones[d] - groupdata.nghostzones[d]);

Replacement in AMReX/src/schedule.cxx at line 255 [27.531]

B:BD[17.4331] → [2.140:184]

    const GridPtrDesc grid(leveldata, mfi);

[17.4331]

[17.4369]

    const GridPtrDesc1 grid(leveldata, groupdata, mfi);

Replacement in AMReX/src/schedule.cxx at line 257 [27.531]

B:BD[17.4440] → [17.4440:4496]

    CCTK_REAL *restrict const ptr = grid.ptr(vars, vi);

[17.4440]

[11.16689]

    const GF3D1<CCTK_REAL> ptr_ = grid.gf3d(vars, vi);

Insertion in AMReX/src/schedule.cxx at line 259 [27.531]
[11.16690]
[17.4497]
```
    where_t where;
```

Replacement in AMReX/src/schedule.cxx at line 261 [27.531]

B:BD[17.4525] → [17.4525:4820]

      if (!valid.valid_bnd) {
        grid.loop_all(groupdata.indextype, [&](const Loop::PointDesc &p) {
          ptr[p.idx] = 0.0 / 0.0;
        });
      } else {
        grid.loop_int(groupdata.indextype, [&](const Loop::PointDesc &p) {
          ptr[p.idx] = 0.0 / 0.0;
        });
      }

[17.4525]

[17.4820]

      if (!valid.valid_bnd)
        where = where_t::everywhere;
      else
        where = where_t::interior;

Replacement in AMReX/src/schedule.cxx at line 266 [27.531]

B:BD[17.4833] → [17.4833:4999]

      if (!valid.valid_bnd) {
        grid.loop_bnd(groupdata.indextype, [&](const Loop::PointDesc &p) {
          ptr[p.idx] = 0.0 / 0.0;
        });
      } else {

[17.4833]

[17.4999]

      if (!valid.valid_bnd)
        where = where_t::boundary;
      else

Deletion in AMReX/src/schedule.cxx at line 270 [27.531]
B:BD[17.5018] → [17.5018:5026]
```
      }
```

Insertion in AMReX/src/schedule.cxx at line 271 [27.531]

[17.5032]

    grid.loop_idx(where, groupdata.indextype, groupdata.nghostzones,
                  [&](const Loop::PointDesc &p) { ptr_(p.I) = 0.0 / 0.0; });

Replacement in AMReX/src/schedule.cxx at line 293 [27.531]
B:BD[28.1269] → [28.1269:1313]
```
    const GridPtrDesc grid(leveldata, mfi);
```
[28.1269]
[28.1313]
```
    const GridPtrDesc1 grid(leveldata, groupdata, mfi);
```

Replacement in AMReX/src/schedule.cxx at line 295 [27.531]

B:BD[28.1390] → [28.1390:1452]

    const CCTK_REAL *restrict const ptr = grid.ptr(vars, vi);

[28.1390]

[17.5038]

    const GF3D1<const CCTK_REAL> ptr_ = grid.gf3d(vars, vi);

Insertion in AMReX/src/schedule.cxx at line 297 [27.531]
[17.5039]
[28.1453]
```
    where_t where;
```

Replacement in AMReX/src/schedule.cxx at line 299 [27.531]

B:BD[28.1480] → [28.1480:1584]

B:BD[28.1584] → [29.53:144]

∅:D[29.144] → [28.1622:1724]

B:BD[28.1622] → [28.1622:1724]

B:BD[28.1724] → [29.145:236]

∅:D[29.236] → [28.1762:1782]

B:BD[28.1762] → [28.1762:1782]

      if (valid.valid_bnd) {
        grid.loop_all(groupdata.indextype, [&](const Loop::PointDesc &p) {
          if (CCTK_BUILTIN_EXPECT(isnan(ptr[p.idx]), false))
            found_nan = true;
        });
      } else {
        grid.loop_int(groupdata.indextype, [&](const Loop::PointDesc &p) {
          if (CCTK_BUILTIN_EXPECT(isnan(ptr[p.idx]), false))
            found_nan = true;
        });
      }

[28.1480]

[28.1782]

      if (valid.valid_bnd)
        where = where_t::everywhere;
      else
        where = where_t::interior;

Replacement in AMReX/src/schedule.cxx at line 304 [27.531]

B:BD[28.1795] → [28.1795:1899]

B:BD[28.1899] → [29.237:328]

∅:D[29.328] → [28.1937:1964]

B:BD[28.1937] → [28.1937:1964]

      if (valid.valid_bnd) {
        grid.loop_bnd(groupdata.indextype, [&](const Loop::PointDesc &p) {
          if (CCTK_BUILTIN_EXPECT(isnan(ptr[p.idx]), false))
            found_nan = true;
        });
      } else {

[28.1795]

[28.1964]

      if (valid.valid_bnd)
        where = where_t::boundary;
      else

Deletion in AMReX/src/schedule.cxx at line 308 [27.531]
B:BD[28.1983] → [28.1983:1991]
```
      }
```

Insertion in AMReX/src/schedule.cxx at line 309 [27.531]

[28.1997]

    grid.loop_idx(where, groupdata.indextype, groupdata.nghostzones,
                  [&](const Loop::PointDesc &p) {
                    if (CCTK_BUILTIN_EXPECT(isnan(ptr_(p.I)), false))
                      found_nan = true;
                  });

Insertion in AMReX/src/schedule.cxx at line 482 [27.531]
[30.2486]
[31.1400]
```
    GridPtrDesc1 grid1(leveldata, groupdata, mfi);
```

Replacement in AMReX/src/schedule.cxx at line 485 [27.531]

B:BD[31.1535] → [26.4319:4374]

∅:D[11.16799] → [12.14892:14969]

B:BD[26.4374] → [12.14892:14969]

∅:D[26.4481] → [31.1637:1645]

∅:D[12.14969] → [31.1637:1645]

B:BD[31.1637] → [31.1637:1645]

      for (int vi = 0; vi < groupdata.numvars; ++vi) {
        cctkGH->data[groupdata.firstvarindex + vi][tl] = grid.ptr(vars, vi);
      }

[31.1535]

[31.1645]

      for (int vi = 0; vi < groupdata.numvars; ++vi)
        cctkGH->data[groupdata.firstvarindex + vi][tl] = grid1.ptr(vars, vi);

Replacement in AMReX/src/schedule.cxx at line 549 [27.531]

B:BD[11.16827] → [11.16827:16899]

    const cGH *restrict threadGH = &thread_local_cctkGH.at(thread_num);

[11.16827]

[11.16899]

    const cGH *restrict threadGH = &thread_local_info.at(thread_num)->cctkGH;

Replacement in AMReX/src/schedule.cxx at line 554 [27.531]

B:BD[27.4062] → [12.16513:16586]

  const TileBox &restrict tilebox = thread_local_tilebox.at(thread_num);

[27.4062]

[12.16586]

  const TileBox &restrict tilebox = thread_local_info.at(thread_num)->tilebox;

Insertion in AMReX/src/schedule.cxx at line 648 [27.531]
[32.1678]
[32.1706]
Replacement in AMReX/src/schedule.cxx at line 650 [27.531]
B:BD[32.1749] → [32.1749:1792]
B:BD[32.1792] → [12.16706:16750]
```
  thread_local_cctkGH.resize(max_threads);
  thread_local_tilebox.resize(max_threads);
```
[32.1749]
[32.1792]
```
  thread_local_info.resize(max_threads);
```

Replacement in AMReX/src/schedule.cxx at line 652 [27.531]

B:BD[32.1834] → [32.1834:1891]

    cGH *restrict threadGH = &thread_local_cctkGH.at(n);

[32.1834]

[32.1891]

    thread_local_info.at(n) = make_unique<thread_local_info_t>();
    cGH *restrict threadGH = &thread_local_info.at(n)->cctkGH;

Insertion in AMReX/src/schedule.cxx at line 657 [27.531]
[32.1988]
[33.941]
```
    thread_local_info.at(n)->mfiter = nullptr;
```

Insertion in AMReX/src/schedule.cxx at line 659 [27.531]

[33.945]

  swap(saved_thread_local_info, thread_local_info);
  assert(thread_local_info.empty());

Insertion in AMReX/src/schedule.cxx at line 898 [27.531]

[31.2071]

[34.3462]

      for (int tl = 0; tl < ntls; ++tl)
        for (int vi = 0; vi < groupdata.numvars; ++vi)
          check_valid(leveldata, groupdata, vi, tl,
                      [&]() { return "CycleTimelevels"; });

Insertion in AMReX/src/schedule.cxx at line 1078 [27.531]

[27.5189]

[27.5238]

    // TODO: provide looping function
    assert(thread_local_info.empty());
    swap(thread_local_info, saved_thread_local_info);

Replacement in AMReX/src/schedule.cxx at line 1084 [27.531]

B:BD[12.17101] → [12.17101:17169]

      cGH *restrict threadGH = &thread_local_cctkGH.at(thread_num);

[12.17101]

[32.2190]

      thread_local_info_t &restrict thread_info =
          *thread_local_info.at(thread_num);
      cGH *restrict threadGH = &thread_info.cctkGH;

Replacement in AMReX/src/schedule.cxx at line 1088 [27.531]

B:BD[32.2229] → [12.17170:17248]

      TileBox &restrict thread_tilebox = thread_local_tilebox.at(thread_num);

[32.2229]

[14.6107]

      TileBox &restrict thread_tilebox = thread_info.tilebox;

Insertion in AMReX/src/schedule.cxx at line 1098 [27.531]
[11.17068]
[12.17375]
```
          thread_info.mfiter = &mfi;
```
Insertion in AMReX/src/schedule.cxx at line 1102 [27.531]
[12.17516]
[34.3938]
```
          thread_info.mfiter = nullptr;
```

Insertion in AMReX/src/schedule.cxx at line 1107 [27.531]

[27.5564]

[27.5612]

    swap(saved_thread_local_info, thread_local_info);
    assert(thread_local_info.empty());

Replacement in AMReX/src/schedule.cxx at line 1325 [27.531]
B:BD[5.4756] → [5.4756:4772]
```
#warning "TODO"
```
[5.4756]
[5.4772]
```
#warning "TODO: scale fluxes correctly"
```

Replacement in AMReX/src/schedule.cxx at line 1401 [27.531]

B:BD[35.161] → [5.5869:5959]

      // Only restrict valid grid functions. Restriction only uses the
      // interior.

[35.161]

[17.8792]

      // Only restrict valid grid functions. Restriction only uses the interior.

Replacement in AMReX/src/schedule.cxx at line 1409 [27.531]

B:BD[17.9112] → [17.9112:9228]

        for (int vi = 0; vi < groupdata.numvars; ++vi) {
          groupdata.valid.at(tl).at(vi).valid_int = false;

[17.9112]

[17.9228]

        for (int vi = 0; vi < groupdata.numvars; ++vi)

Deletion in AMReX/src/schedule.cxx at line 1411 [27.531]
B:BD[17.9287] → [17.9287:9297]
```
        }
```

Replacement in AMReX/src/schedule.cxx at line 1427 [27.531]

B:BD[17.9559] → [17.9559:9620]

        if (groupdata.indextype == array<int, dim>{0, 0, 0})

[17.9559]

[28.3375]

#warning                                                                       \
    "TODO: Allow different restriction operators, and ensure this is conservative"
        // rank: 0: vertex, 1: edge, 2: face, 3: volume
        int rank = 0;
        for (int d = 0; d < dim; ++d)
          rank += groupdata.indextype[d];
        switch (rank) {
        case 0:

Replacement in AMReX/src/schedule.cxx at line 1437 [27.531]

∅:D[28.3495] → [17.9741:9943]

B:BD[17.9741] → [17.9741:9943]

        else if (groupdata.indextype == array<int, dim>{1, 0, 0} ||
                 groupdata.indextype == array<int, dim>{0, 1, 0} ||
                 groupdata.indextype == array<int, dim>{0, 0, 1})

[28.3495]

[28.3496]

          break;
        case 1:

Replacement in AMReX/src/schedule.cxx at line 1441 [27.531]

∅:D[28.3616] → [17.10064:10266]

B:BD[17.10064] → [17.10064:10266]

        else if (groupdata.indextype == array<int, dim>{1, 1, 0} ||
                 groupdata.indextype == array<int, dim>{1, 0, 1} ||
                 groupdata.indextype == array<int, dim>{0, 1, 1})

[28.3616]

[28.3617]

          break;
        case 2:

Replacement in AMReX/src/schedule.cxx at line 1445 [27.531]
∅:D[28.3737] → [17.10387:10453]
B:BD[17.10387] → [17.10387:10453]
```
        else if (groupdata.indextype == array<int, dim>{1, 1, 1})
```
[28.3737]
[28.3738]
```
          break;
        case 3:
```
Replacement in AMReX/src/schedule.cxx at line 1449 [27.531]
B:BD[17.10584] → [17.10584:10597]
```
        else
```
[17.10584]
[17.10597]
```
          break;
        default:
```
Insertion in AMReX/src/schedule.cxx at line 1452 [27.531]
[17.10618]
[20.12936]
```
        }
```
Insertion in AMReX/src/schedule.hxx at line 62 [27.5992]
[12.18247]
[12.18247]
```
// TODO: remove this
```

Insertion in AMReX/src/schedule.hxx at line 76 [27.5992]

[12.18741]

  }
};
struct GridPtrDesc1 : GridDesc {
  Dim3 cactus_offset;
  array<int, dim> gimin, gimax;
  array<int, dim> gash;
  GridPtrDesc1() = delete;
  GridPtrDesc1(const GHExt::LevelData &leveldata,
               const GHExt::LevelData::GroupData &groupdata, const MFIter &mfi);
  template <typename T> T *ptr(const Array4<T> &vars, int vi) const {
    return vars.ptr(cactus_offset.x + gimin[0], cactus_offset.y + gimin[1],
                    cactus_offset.z + gimin[2], vi);
  }
  template <typename T>
  T &idx(const Array4<T> &vars, int i, int j, int k, int vi) const {
    return vars(cactus_offset.x + gimin[0] + i, cactus_offset.y + gimin[1] + i,
                cactus_offset.z + gimin[2] + j, vi);
  }
  template <typename T> GF3D1<T> gf3d(const Array4<T> &vars, int vi) const {
    return GF3D1<T>(ptr(vars, vi), gimin, gimax, gash);

Replacement in AMReX/src/schedule.hxx at line 106 [27.5992]

B:BD[17.11179] → [7.2855:3014]

void check_valid(
    const GHExt::LevelData &leveldata,
    const GHExt::LevelData::GroupData &groupdata, int vi, int tl,
    const function<string()> &msg);

[17.11179]

[12.18748]

void check_valid(const GHExt::LevelData &leveldata,
                 const GHExt::LevelData::GroupData &groupdata, int vi, int tl,
                 const function<string()> &msg);

Insertion in HydroToyAMReX/interface.ccl at line 24 [37.5609]
[36.201]
[36.201]
```
  eint
```

Replacement in HydroToyAMReX/interface.ccl at line 27 [37.5609]

B:BD[36.226] → [10.93:182]

CCTK_REAL flux_x TYPE=gf TIMELEVELS=1 TAGS='index={0 1 1} checkpoint="no" restrict="no"'

[36.226]

[36.285]

CCTK_REAL flux_x TYPE=gf TIMELEVELS=1 TAGS='index={0 1 1} checkpoint="no" restrict="no" nghostzones={0 0 0}'

Replacement in HydroToyAMReX/interface.ccl at line 34 [37.5609]

B:BD[37.5797] → [10.183:272]

CCTK_REAL flux_y TYPE=gf TIMELEVELS=1 TAGS='index={1 0 1} checkpoint="no" restrict="no"'

[37.5797]

[37.5837]

CCTK_REAL flux_y TYPE=gf TIMELEVELS=1 TAGS='index={1 0 1} checkpoint="no" restrict="no" nghostzones={0 0 0}'

Replacement in HydroToyAMReX/interface.ccl at line 41 [37.5609]

B:BD[37.5870] → [10.273:362]

CCTK_REAL flux_z TYPE=gf TIMELEVELS=1 TAGS='index={1 1 0} checkpoint="no" restrict="no"'

[37.5870]

[37.5908]

CCTK_REAL flux_z TYPE=gf TIMELEVELS=1 TAGS='index={1 1 0} checkpoint="no" restrict="no" nghostzones={0 0 0}'

Insertion in HydroToyAMReX/param.ccl at line 30 [37.5941]

[8.3]

CCTK_INT output_every "Calculate output quantities every that many iterations" STEERABLE=recover
{
  0:* :: ""
} 1

Replacement in HydroToyAMReX/schedule.ccl at line 39 [37.6006]
B:BD[37.6149] → [8.75:89]
```
if (output) {
```
[37.6149]
[8.89]
```
# TODO: Use a more accurate condition
if (output || output_every > 0) {
```
Insertion in HydroToyAMReX/schedule.ccl at line 51 [37.6006]
[8.362]
[8.362]

Replacement in HydroToyAMReX/schedule.ccl at line 60 [37.6006]

B:BD[38.986] → [8.645:934]

    SCHEDULE HydroToyAMReX_Output IN HydroToyAMReX_OutputGroup AFTER HydroToyAMReX_Fluxes
    {
      LANG: C
      READS: conserved(interior)
      READS: flux_x(interior) flux_y(interior) flux_z(interior)
      READS: AMReX::regrid_error(interior)
    } "Output grid data"
} # if output

[38.986]

[38.1247]

    if (output) {
      SCHEDULE HydroToyAMReX_Output IN HydroToyAMReX_OutputGroup AFTER HydroToyAMReX_Fluxes
      {
        LANG: C
        READS: conserved(interior)
        READS: flux_x(interior) flux_y(interior) flux_z(interior)
        READS: AMReX::regrid_error(interior)
      } "Output grid data"
    } # if output
}

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 19 [37.6253]

[36.3961]

[37.6562]

template <typename T> inline T fmax5(T x0, T x1, T x2, T x3, T x4) {
  T r0 = fmax(x0, x1);
  T r1 = fmax(x2, x3);
  T r2 = fmax(x4, r0);
  T r3 = fmax(r1, r2);
  return r3;
}
////////////////////////////////////////////////////////////////////////////////

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 33 [37.6253]

[36.4096]

  const Loop::GF3D<CCTK_REAL, 1, 1, 1> rho_(cctkGH, rho);
  const Loop::GF3D<CCTK_REAL, 1, 1, 1> momx_(cctkGH, momx);
  const Loop::GF3D<CCTK_REAL, 1, 1, 1> momy_(cctkGH, momy);
  const Loop::GF3D<CCTK_REAL, 1, 1, 1> momz_(cctkGH, momz);
  const Loop::GF3D<CCTK_REAL, 1, 1, 1> etot_(cctkGH, etot);

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 42 [37.6253]

∅:D[39.3333] → [36.4208:4332]

B:BD[36.4208] → [36.4208:4332]

      rho[p.idx] = 1.0;
      momx[p.idx] = 0.0;
      momy[p.idx] = 0.0;
      momz[p.idx] = 0.0;
      etot[p.idx] = 1.0;

[39.3333]

[36.4332]

      rho_(p.I) = 1.0;
      momx_(p.I) = 0.0;
      momy_(p.I) = 0.0;
      momz_(p.I) = 0.0;
      etot_(p.I) = 1.0;

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 52 [37.6253]

∅:D[39.3402] → [36.4459:4588]

B:BD[36.4459] → [36.4459:4588]

B:BD[36.4588] → [39.3403:3462]

      rho[p.idx] = 1.0;
      momx[p.idx] = 0.0 + amplitude * sin(M_PI * p.x);
      momy[p.idx] = 0.0;
      momz[p.idx] = 0.0;
      etot[p.idx] = 1.0; // should add kinetic energy here

[39.3402]

[39.3462]

      rho_(p.I) = 1.0;
      momx_(p.I) = 0.0 + amplitude * sin(M_PI * p.x);
      momy_(p.I) = 0.0;
      momz_(p.I) = 0.0;
      etot_(p.I) = 1.0; // should add kinetic energy here

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 63 [37.6253]

B:BD[39.3613] → [39.3613:3747]

        rho[p.idx] = 2.0;
        momx[p.idx] = 0.0;
        momy[p.idx] = 0.0;
        momz[p.idx] = 0.0;
        etot[p.idx] = 2.0;

[39.3613]

[39.3747]

        rho_(p.I) = 2.0;
        momx_(p.I) = 0.0;
        momy_(p.I) = 0.0;
        momz_(p.I) = 0.0;
        etot_(p.I) = 2.0;

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 69 [37.6253]

B:BD[39.3762] → [39.3762:3896]

        rho[p.idx] = 1.0;
        momx[p.idx] = 0.0;
        momy[p.idx] = 0.0;
        momz[p.idx] = 0.0;
        etot[p.idx] = 1.0;

[39.3762]

[39.3896]

        rho_(p.I) = 1.0;
        momx_(p.I) = 0.0;
        momy_(p.I) = 0.0;
        momz_(p.I) = 0.0;
        etot_(p.I) = 1.0;

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 82 [37.6253]

B:BD[39.4138] → [39.4138:4272]

        rho[p.idx] = 2.0;
        momx[p.idx] = 0.0;
        momy[p.idx] = 0.0;
        momz[p.idx] = 0.0;
        etot[p.idx] = 2.0;

[39.4138]

[39.4272]

        rho_(p.I) = 2.0;
        momx_(p.I) = 0.0;
        momy_(p.I) = 0.0;
        momz_(p.I) = 0.0;
        etot_(p.I) = 2.0;

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 88 [37.6253]

B:BD[39.4287] → [39.4287:4421]

        rho[p.idx] = 1.0;
        momx[p.idx] = 0.0;
        momy[p.idx] = 0.0;
        momz[p.idx] = 0.0;
        etot[p.idx] = 1.0;

[39.4287]

[39.4421]

        rho_(p.I) = 1.0;
        momx_(p.I) = 0.0;
        momy_(p.I) = 0.0;
        momz_(p.I) = 0.0;
        etot_(p.I) = 1.0;

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 135 [37.6253]

[37.6674]

[36.4713]


  const Loop::GF3D<const CCTK_REAL, 1, 1, 1> rho_(cctkGH, rho);
  const Loop::GF3D<const CCTK_REAL, 1, 1, 1> momx_(cctkGH, momx);
  const Loop::GF3D<const CCTK_REAL, 1, 1, 1> momy_(cctkGH, momy);
  const Loop::GF3D<const CCTK_REAL, 1, 1, 1> momz_(cctkGH, momz);
  const Loop::GF3D<const CCTK_REAL, 1, 1, 1> etot_(cctkGH, etot);
  const Loop::GF3D<CCTK_REAL, 1, 1, 1> press_(cctkGH, press);
  const Loop::GF3D<CCTK_REAL, 1, 1, 1> velx_(cctkGH, velx);
  const Loop::GF3D<CCTK_REAL, 1, 1, 1> vely_(cctkGH, vely);
  const Loop::GF3D<CCTK_REAL, 1, 1, 1> velz_(cctkGH, velz);
  const Loop::GF3D<CCTK_REAL, 1, 1, 1> eint_(cctkGH, eint);

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 154 [37.6253]

B:BD[36.4880] → [36.4880:5066]

        sqrt(pow(momx[p.idx], 2) + pow(momy[p.idx], 2) + pow(momz[p.idx], 2)) /
        (2 * rho[p.idx]);
    CCTK_REAL eint = etot[p.idx] - ekin;
    press[p.idx] = (gamma - 1) * eint;

[36.4880]

[37.6674]

        sqrt(pow(momx_(p.I), 2) + pow(momy_(p.I), 2) + pow(momz_(p.I), 2)) /
        (2 * rho_(p.I));
    eint_(p.I) = etot_(p.I) - ekin;

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 158 [37.6253]

B:BD[37.6675] → [36.5067:5199]

    velx[p.idx] = momx[p.idx] / rho[p.idx];
    vely[p.idx] = momy[p.idx] / rho[p.idx];
    velz[p.idx] = momz[p.idx] / rho[p.idx];

[37.6675]

[36.5199]

    press_(p.I) = (gamma - 1) * eint_(p.I);
    velx_(p.I) = momx_(p.I) / rho_(p.I);
    vely_(p.I) = momy_(p.I) / rho_(p.I);
    velz_(p.I) = momz_(p.I) / rho_(p.I);

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 222 [37.6253]
[36.7389]
[36.7389]
```
#if 0
```

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 240 [37.6253]

[36.8345]

#else
  const array<int, dim> nghosts{0, 0, 0};
  const Loop::GF3D1<CCTK_REAL> fxrho_(cctkGH, {0, 1, 1}, nghosts, fxrho);
  const Loop::GF3D1<CCTK_REAL> fxmomx_(cctkGH, {0, 1, 1}, nghosts, fxmomx);
  const Loop::GF3D1<CCTK_REAL> fxmomy_(cctkGH, {0, 1, 1}, nghosts, fxmomy);
  const Loop::GF3D1<CCTK_REAL> fxmomz_(cctkGH, {0, 1, 1}, nghosts, fxmomz);
  const Loop::GF3D1<CCTK_REAL> fxetot_(cctkGH, {0, 1, 1}, nghosts, fxetot);
  const Loop::GF3D1<CCTK_REAL> fyrho_(cctkGH, {1, 0, 1}, nghosts, fyrho);
  const Loop::GF3D1<CCTK_REAL> fymomx_(cctkGH, {1, 0, 1}, nghosts, fymomx);
  const Loop::GF3D1<CCTK_REAL> fymomy_(cctkGH, {1, 0, 1}, nghosts, fymomy);
  const Loop::GF3D1<CCTK_REAL> fymomz_(cctkGH, {1, 0, 1}, nghosts, fymomz);
  const Loop::GF3D1<CCTK_REAL> fyetot_(cctkGH, {1, 0, 1}, nghosts, fyetot);

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 254 [37.6253]

[36.8346]

  const Loop::GF3D1<CCTK_REAL> fzrho_(cctkGH, {1, 1, 0}, nghosts, fzrho);
  const Loop::GF3D1<CCTK_REAL> fzmomx_(cctkGH, {1, 1, 0}, nghosts, fzmomx);
  const Loop::GF3D1<CCTK_REAL> fzmomy_(cctkGH, {1, 1, 0}, nghosts, fzmomy);
  const Loop::GF3D1<CCTK_REAL> fzmomz_(cctkGH, {1, 1, 0}, nghosts, fzmomz);
  const Loop::GF3D1<CCTK_REAL> fzetot_(cctkGH, {1, 1, 0}, nghosts, fzetot);
#endif

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 353 [37.6253]
[39.5723]
[39.5723]
```
#if 0
```

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 371 [37.6253]

[39.6769]

#else
  const array<int, dim> nghosts{0, 0, 0};
  const Loop::GF3D1<const CCTK_REAL> fxrho_(cctkGH, {0, 1, 1}, nghosts, fxrho);
  const Loop::GF3D1<const CCTK_REAL> fxmomx_(cctkGH, {0, 1, 1}, nghosts,
                                             fxmomx);
  const Loop::GF3D1<const CCTK_REAL> fxmomy_(cctkGH, {0, 1, 1}, nghosts,
                                             fxmomy);
  const Loop::GF3D1<const CCTK_REAL> fxmomz_(cctkGH, {0, 1, 1}, nghosts,
                                             fxmomz);
  const Loop::GF3D1<const CCTK_REAL> fxetot_(cctkGH, {0, 1, 1}, nghosts,
                                             fxetot);
  const Loop::GF3D1<const CCTK_REAL> fyrho_(cctkGH, {1, 0, 1}, nghosts, fyrho);
  const Loop::GF3D1<const CCTK_REAL> fymomx_(cctkGH, {1, 0, 1}, nghosts,
                                             fymomx);
  const Loop::GF3D1<const CCTK_REAL> fymomy_(cctkGH, {1, 0, 1}, nghosts,
                                             fymomy);
  const Loop::GF3D1<const CCTK_REAL> fymomz_(cctkGH, {1, 0, 1}, nghosts,
                                             fymomz);
  const Loop::GF3D1<const CCTK_REAL> fyetot_(cctkGH, {1, 0, 1}, nghosts,
                                             fyetot);

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 393 [37.6253]

[39.6770]

  const Loop::GF3D1<const CCTK_REAL> fzrho_(cctkGH, {1, 1, 0}, nghosts, fzrho);
  const Loop::GF3D1<const CCTK_REAL> fzmomx_(cctkGH, {1, 1, 0}, nghosts,
                                             fzmomx);
  const Loop::GF3D1<const CCTK_REAL> fzmomy_(cctkGH, {1, 1, 0}, nghosts,
                                             fzmomy);
  const Loop::GF3D1<const CCTK_REAL> fzmomz_(cctkGH, {1, 1, 0}, nghosts,
                                             fzmomz);
  const Loop::GF3D1<const CCTK_REAL> fzetot_(cctkGH, {1, 1, 0}, nghosts,
                                             fzetot);
#endif

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 443 [37.6253]

[40.5365]

[36.11950]


  if (false) {
    Loop::loop_int<1, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      regrid_error_(p.I) = fabs(p.x) < 0.2;
    });
  }

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 462 [37.6253]

B:BD[37.7435] → [40.5724:5891]

    regrid_error_(p.I) =
        fmax(fmax(fmax(fmax(calcerr(rho_), calcerr(momx_)), calcerr(momy_)),
                  calcerr(momz_)),
             calcerr(etot_));

[37.7435]

[37.8055]

    regrid_error_(p.I) = fmax5(calcerr(rho_), calcerr(momx_), calcerr(momy_),
                               calcerr(momz_), calcerr(etot_));

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 480 [37.6253]
[8.1077]
[39.8292]
```
#if 0
```

Insertion in HydroToyAMReX/src/hydrotoy.cxx at line 486 [37.6253]

[39.8640]

#else
  const array<int, dim> nghosts{0, 0, 0};
  const Loop::GF3D1<const CCTK_REAL> fxrho_(cctkGH, {0, 1, 1}, nghosts, fxrho);
  const Loop::GF3D1<const CCTK_REAL> fxmomx_(cctkGH, {0, 1, 1}, nghosts,
                                             fxmomx);
  const Loop::GF3D1<const CCTK_REAL> fxmomy_(cctkGH, {0, 1, 1}, nghosts,
                                             fxmomy);
  const Loop::GF3D1<const CCTK_REAL> fxmomz_(cctkGH, {0, 1, 1}, nghosts,
                                             fxmomz);
  const Loop::GF3D1<const CCTK_REAL> fxetot_(cctkGH, {0, 1, 1}, nghosts,
                                             fxetot);
#endif

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 507 [37.6253]
B:BD[39.8782] → [39.8782:8790]
```
      {
```
[39.8782]
[8.1185]
```
  {
```
Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 519 [37.6253]
∅:D[8.1617] → [39.9009:9021]
B:BD[39.9009] → [39.9009:9021]
```
    }
  });
```
[8.1617]
[39.9021]
```
      }
    });
```

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 522 [37.6253]

B:BD[39.9022] → [39.9022:9120]

  Loop::loop_all<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
    if (p.j == 0 && p.k == 0) {

[39.9022]

[39.9171]

    Loop::loop_all<0, 1, 1>(cctkGH, [&](const Loop::PointDesc &p) {
      if (p.j == 0 && p.k == 0) {

Replacement in HydroToyAMReX/src/hydrotoy.cxx at line 530 [37.6253]
B:BD[39.9421] → [39.9421:9427]
```
  });
```
[39.9415]
[8.1771]
```
    });
```