eschnett/cactusamrex - Change UUGQGVC4WEKN64WAP7F5QPS2UHGQB5ZLMFRIYNWKMIEBDO3QRX4AC

Overload evolution manager

Created by Erik Schnetter on July 12, 2019

UUGQGVC4WEKN64WAP7F5QPS2UHGQB5ZLMFRIYNWKMIEBDO3QRX4AC

Dependencies

In channels

master

main

Change contents

Insertion in AMReX/param.ccl at line 2 [3.17918]

[3.17959]


SHARES: Cactus
USES BOOLEAN terminate_next
USES KEYWORD terminate
USES CCTK_INT cctk_itlast
USES CCTK_REAL cctk_initial_time
USES CCTK_REAL cctk_final_time
USES CCTK_REAL max_runtime

Insertion in AMReX/src/AMReX.hxx at line 14 [3.15275]
[3.15516]
[3.15516]
```
#include <vector>
```
Deletion in AMReX/src/AMReX.hxx at line 36 [3.15275]
B:BD[3.15855] → [3.15855:15872]
```
  MultiFab mfab;
```

Replacement in AMReX/src/AMReX.hxx at line 37 [3.15275]

B:BD[2.1] → [2.1:26]

  // TODO: use StateData

[2.1]

[3.15872]

  struct GroupData {
    int firstvarindex;
    int numvars;
    int numtimelevels;
    MultiFab mfab;
  };
  vector<GroupData> groupdata;

Insertion in AMReX/src/driver.cxx at line 58 [3.15972]

[4.887]


  CCTK_OverloadInitialise(Initialise);
  CCTK_OverloadEvolve(Evolve);
  CCTK_OverloadShutdown(Shutdown);

Deletion in AMReX/src/driver.cxx at line 109 [3.15972]
B:BD[3.17248] → [3.17248:17249]
B:BD[3.17249] → [5.102:196]
∅:D[5.196] → [3.17249:17300]
B:BD[3.17249] → [3.17249:17300]
```
  CCTK_VINFO("Groups: %d", CCTK_NumGroups());
  CCTK_VINFO("Variables: %d", CCTK_NumVars());
  const int nvars = 4; // number of grid functions
```

Replacement in AMReX/src/driver.cxx at line 113 [3.15972]

B:BD[3.17362] → [3.17362:17459]

  // Allocate grid hierarchy
  ghext->mfab = MultiFab(ghext->ba, dm, nvars, ghext->nghostzones);

[3.17362]

[3.17459]

  const int numgroups = CCTK_NumGroups();
  ghext->groupdata.resize(numgroups);
  for (int gi = 0; gi < numgroups; ++gi) {
    cGroup group;
    int ierr = CCTK_GroupData(gi, &group);
    assert(!ierr);
    assert(group.grouptype == CCTK_GF);
    assert(group.vartype == CCTK_VARIABLE_REAL);
    assert(group.disttype == CCTK_DISTRIB_DEFAULT);
    assert(group.dim == dim);
    GHExt::GroupData &groupdata = ghext->groupdata.at(gi);
    groupdata.firstvarindex = CCTK_FirstVarIndexI(gi);
    groupdata.numvars = group.numvars;
    groupdata.numtimelevels = group.numtimelevels;
    // Allocate grid hierarchy
    groupdata.mfab =
        MultiFab(ghext->ba, dm, groupdata.numvars * groupdata.numtimelevels,
                 ghext->nghostzones);
  }

Insertion in AMReX/src/schedule.cxx at line 8 [6.531]

[6.680]

#include <cctki_GHExtensions.h>
#include <cctki_ScheduleBindings.h>
#include <cctki_WarnLevel.h>

Insertion in AMReX/src/schedule.cxx at line 17 [6.531]
[6.766]
[6.766]
```
#include <sys/time.h>
```

Replacement in AMReX/src/schedule.cxx at line 128 [6.531]

B:BD[2.792] → [2.792:947]

  const Array4<CCTK_REAL> &vars = ghext->mfab.array(mfi);
  for (int vi = 0; vi < 4; ++vi)
    cctkGH->data[vi][0] = vars.ptr(imin.x, imin.y, imin.z, vi);

[2.792]

[6.3873]

  for (auto &restrict groupdata : ghext->groupdata) {
    const Array4<CCTK_REAL> &vars = groupdata.mfab.array(mfi);
    for (int tl = 0; tl < groupdata.numtimelevels; ++tl)
      for (int n = 0; n < groupdata.numvars; ++n)
        cctkGH->data[groupdata.firstvarindex + n][tl] =
            vars.ptr(imin.x, imin.y, imin.z, tl * groupdata.numvars + n);
  }

Replacement in AMReX/src/schedule.cxx at line 148 [6.531]

B:BD[2.1200] → [2.1200:1302]

  int nvars = CCTK_NumVars();
  for (int vi = 0; vi < nvars; ++vi)
    cctkGH->data[vi][0] = nullptr;

[2.1200]

[6.4059]

  for (auto &restrict groupdata : ghext->groupdata) {
    for (int tl = 0; tl < groupdata.numtimelevels; ++tl)
      for (int n = 0; n < groupdata.numvars; ++n)
        cctkGH->data[groupdata.firstvarindex + n][tl] = nullptr;
  }

Insertion in AMReX/src/schedule.cxx at line 176 [6.531]

[6.4682]

// Schedule initialisation
int Initialise(tFleshConfig *config) {
  cGH *restrict const cctkGH = CCTK_SetupGH(config, 0);
  CCTKi_AddGH(config, 0, cctkGH);
  // Initialise iteration and time
  cctkGH->cctk_iteration = 0;
  cctkGH->cctk_time = *static_cast<const CCTK_REAL *>(
      CCTK_ParameterGet("cctk_initial_time", "Cactus", nullptr));
  // Initialise schedule
  CCTKi_ScheduleGHInit(cctkGH);
  // Initialise all grid extensions
  CCTKi_InitGHExtensions(cctkGH);
  CCTK_Traverse(cctkGH, "CCTK_WRAGH");
  CCTK_Traverse(cctkGH, "CCTK_PARAMCHECK");
  CCTKi_FinaliseParamWarn();
  CCTK_Traverse(cctkGH, "CCTK_BASEGRID");
  const char *recovery_mode = *static_cast<const char *const *>(
      CCTK_ParameterGet("recovery_mode", "Cactus", nullptr));
  if (!config->recovered || !CCTK_Equals(recovery_mode, "strict")) {
    // Set up initial conditions
    CCTK_Traverse(cctkGH, "CCTK_INITIAL");
    CCTK_Traverse(cctkGH, "CCTK_POSTINITIAL");
    CCTK_Traverse(cctkGH, "CCTK_POSTPOSTINITIAL");
    CCTK_Traverse(cctkGH, "CCTK_POSTSTEP");
  }
  if (config->recovered) {
    // Recover
    CCTK_Traverse(cctkGH, "CCTK_RECOVER_VARIABLES");
    CCTK_Traverse(cctkGH, "CCTK_POST_RECOVER_VARIABLES");
  }
  // Checkpoint, analysis, output
  CCTK_Traverse(cctkGH, "CCTK_CPINITIAL");
  CCTK_Traverse(cctkGH, "CCTK_ANALYSIS");
  CCTK_OutputGH(cctkGH);
  return 0;
}
bool EvolutionIsDone(cGH *restrict const cctkGH) {
  DECLARE_CCTK_PARAMETERS;
  static timeval starttime = {0, 0};
  // On the first time through, get the start time
  if (starttime.tv_sec == 0 && starttime.tv_usec == 0)
    gettimeofday(&starttime, nullptr);
  if (terminate_next || CCTK_TerminationReached(cctkGH))
    return true;
  if (CCTK_Equals(terminate, "never"))
    return false;
  bool max_iteration_reached = cctkGH->cctk_iteration >= cctk_itlast;
  bool max_simulation_time_reached = cctk_initial_time < cctk_final_time
                                         ? cctkGH->cctk_time >= cctk_final_time
                                         : cctkGH->cctk_time <= cctk_final_time;
  // Get the elapsed runtime in minutes and compare with max_runtime
  timeval currenttime;
  gettimeofday(&currenttime, NULL);
  bool max_runtime_reached =
      CCTK_REAL(currenttime.tv_sec - starttime.tv_sec) / 60 >= max_runtime;
  if (CCTK_Equals(terminate, "iteration"))
    return max_iteration_reached;
  if (CCTK_Equals(terminate, "time"))
    return max_simulation_time_reached;
  if (CCTK_Equals(terminate, "runtime"))
    return max_runtime_reached;
  if (CCTK_Equals(terminate, "any"))
    return max_iteration_reached || max_simulation_time_reached ||
           max_runtime_reached;
  if (CCTK_Equals(terminate, "all"))
    return max_iteration_reached && max_simulation_time_reached &&
           max_runtime_reached;
  if (CCTK_Equals(terminate, "either"))
    return max_iteration_reached || max_simulation_time_reached;
  if (CCTK_Equals(terminate, "both"))
    return max_iteration_reached && max_simulation_time_reached;
  assert(0);
}
void CycleTimelevels(cGH *restrict const cctkGH) {
  for (auto &restrict groupdata : ghext->groupdata) {
    for (int tl = groupdata.numtimelevels - 1; tl > 0; --tl)
      MultiFab::Copy(groupdata.mfab, groupdata.mfab,
                     (tl - 1) * groupdata.numvars, tl * groupdata.numvars,
                     groupdata.numvars, ghext->nghostzones);
  }
}
// Schedule evolution
int Evolve(tFleshConfig *config) {
  assert(config);
  cGH *restrict const cctkGH = config->GH[0];
  assert(cctkGH);
  while (!EvolutionIsDone(cctkGH)) {
    cctkGH->cctk_iteration += 1;
    cctkGH->cctk_time += cctkGH->cctk_delta_time;
    CycleTimelevels(cctkGH);
    CCTK_Traverse(cctkGH, "CCTK_PRESTEP");
    CCTK_Traverse(cctkGH, "CCTK_EVOL");
    CCTK_Traverse(cctkGH, "CCTK_POSTSTEP");
    CCTK_Traverse(cctkGH, "CCTK_CHECKPOINT");
    CCTK_Traverse(cctkGH, "CCTK_ANALYSIS");
    CCTK_OutputGH(cctkGH);
  }
  return 0;
}
// Schedule shutdown
int Shutdown(tFleshConfig *config) {
  assert(config);
  cGH *restrict const cctkGH = config->GH[0];
  assert(cctkGH);
  CCTK_Traverse(cctkGH, "CCTK_TERMINATE");
  CCTK_Traverse(cctkGH, "CCTK_SHUTDOWN");
  return 0;
}

Insertion in AMReX/src/schedule.cxx at line 329 [6.531]

[6.5189]

[6.5238]

    assert(!ghext->groupdata.empty());
    MultiFab &mfab = ghext->groupdata.at(0).mfab;
    auto mfitinfo = MFItInfo().SetDynamic(true).EnableTiling({1024000, 16, 32});

Replacement in AMReX/src/schedule.cxx at line 333 [6.531]

B:BD[6.5259] → [6.5259:5425]

    for (MFIter mfi(ghext->mfab, MFItInfo().SetDynamic(true).EnableTiling(
                                     {1024000, 16, 32}));
         mfi.isValid(); ++mfi) {

[6.5259]

[6.5469]

    for (MFIter mfi(mfab, mfitinfo); mfi.isValid(); ++mfi) {

Deletion in AMReX/src/schedule.cxx at line 335 [6.531]

B:BD[6.5506] → [2.1303:1949]

      // CCTK_VINFO("gsh=[%d,%d,%d]", cctkGH->cctk_gsh[0], cctkGH->cctk_gsh[1],
      //            cctkGH->cctk_gsh[2]);
      // CCTK_VINFO("ash=[%d,%d,%d]", cctkGH->cctk_ash[0], cctkGH->cctk_ash[1],
      //            cctkGH->cctk_ash[2]);
      // CCTK_VINFO("lbnd=[%d,%d,%d]", cctkGH->cctk_lbnd[0], cctkGH->cctk_lbnd[1],
      //            cctkGH->cctk_lbnd[2]);
      // CCTK_VINFO("lsh=[%d,%d,%d]", cctkGH->cctk_lsh[0], cctkGH->cctk_lsh[1],
      //            cctkGH->cctk_lsh[2]);
      // CCTK_VINFO("nghostzones=[%d,%d,%d]", cctkGH->cctk_nghostzones[0],
      //            cctkGH->cctk_nghostzones[1], cctkGH->cctk_nghostzones[2]);

Insertion in AMReX/src/schedule.hxx at line 16 [6.5992]

[6.6433]


int Initialise(tFleshConfig *config);
int Evolve(tFleshConfig *config);
int Shutdown(tFleshConfig *config);

Insertion in AMReXTest/interface.ccl at line 6 [7.4807]

[7.4906]


CCTK_REAL check TYPE=gf
{
  check
} "Grid function for index checking"

Replacement in AMReXTest/src/check_loops.cxx at line 36 [7.5134]

B:BD[7.6039] → [7.6039:6099]

      CCTK_REAL *restrict const phi = vars.ptr(0, 0, 0, 0);

[7.6039]

[7.6099]

      CCTK_REAL *restrict const check = vars.ptr(0, 0, 0, 0);

Replacement in AMReXTest/src/check_loops.cxx at line 42 [7.5134]
B:BD[7.6297] → [7.6297:6323]
```
            phi[idx] = 0;
```
[7.6297]
[7.6323]
```
            check[idx] = 0;
```
Replacement in AMReXTest/src/check_loops.cxx at line 71 [7.5134]
B:BD[7.7274] → [7.7274:7318]
```
    atomic<CCTK_REAL> *restrict const phi =
```
[7.7274]
[7.7318]
```
    atomic<CCTK_REAL> *restrict const check =
```
Replacement in AMReXTest/src/check_loops.cxx at line 79 [7.5134]
B:BD[7.7576] → [7.7576:7606]
```
          // phi[idx] += 1.0;
```
[7.7576]
[7.7606]
```
          // check[idx] += 1.0;
```

Replacement in AMReXTest/src/check_loops.cxx at line 81 [7.5134]

B:BD[7.7642] → [5.1443:1522]

          /*bool success = */ phi[idx].compare_exchange_strong(expected, 1.0);

[7.7642]

[7.7716]

          /*bool success = */ check[idx].compare_exchange_strong(expected, 1.0);

Replacement in AMReXTest/src/check_loops.cxx at line 101 [7.5134]

B:BD[7.8262] → [7.8262:8320]

    CCTK_REAL *restrict const phi = vars.ptr(0, 0, 0, 0);

[7.8262]

[7.8320]

    CCTK_REAL *restrict const check = vars.ptr(0, 0, 0, 0);

Replacement in AMReXTest/src/check_loops.cxx at line 107 [7.5134]
B:BD[7.8510] → [7.8510:8543]
```
          assert(phi[idx] == 1);
```
[7.8510]
[7.8543]
```
          assert(check[idx] == 1);
```

Replacement in WaveToyAMReX/interface.ccl at line 9 [3.10196]

B:BD[5.1526] → [5.1526:1616]

# CCTK_REAL phi TYPE=gf TIMELEVELS=3
# {
#   phi
# } "Scalar potential for wave equation"

[5.1526]

[5.1616]

CCTK_REAL phi TYPE=gf TIMELEVELS=3
{
  phi
} "Scalar potential for wave equation"

Replacement in WaveToyAMReX/interface.ccl at line 14 [3.10196]
B:BD[5.1617] → [2.1982:2004]
```
CCTK_REAL phi TYPE=gf
```
[5.1617]
[2.2004]
```
CCTK_REAL err TYPE=gf
```
Deletion in WaveToyAMReX/interface.ccl at line 16 [3.10196]
B:BD[2.2006] → [2.2006:2030]
```
  phi
  phi_p
  phi_p_p
```
Replacement in WaveToyAMReX/interface.ccl at line 17 [3.10196]
B:BD[2.2036] → [2.2036:2075]
```
} "Scalar potential for wave equation"
```
[2.2036]
```
} "Error in scalar potential for wave equation"
```
Deletion in WaveToyAMReX/schedule.ccl at line 13 [3.9594]
B:BD[3.9832] → [3.9832:9833]
B:BD[3.9833] → [5.1804:1889]
```
SCHEDULE WaveToyAMReX_Cycle AT prestep
{
  LANG: C
  OPTIONS: level
} "Advance time"
```
Insertion in WaveToyAMReX/src/wavetoy.cxx at line 13 [3.37]
[3.215]
[3.233]
```
#include <cctype>
```
Insertion in WaveToyAMReX/src/wavetoy.cxx at line 15 [3.37]
[3.250]
[3.250]
```
#include <memory>
#include <regex>
#include <sstream>
#include <string>
```

Deletion in WaveToyAMReX/src/wavetoy.cxx at line 89 [3.37]

B:BD[5.3764] → [5.3764:4059]

B:BD[5.4059] → [2.3280:3372]

}
extern "C" void WaveToyAMReX_Cycle(CCTK_ARGUMENTS) {
  DECLARE_CCTK_ARGUMENTS;
  DECLARE_CCTK_PARAMETERS;
  // Cycle time levels
  MultiFab::Copy(ghext->mfab, ghext->mfab, 1, 2, 1, ghext->nghostzones);
  MultiFab::Copy(ghext->mfab, ghext->mfab, 0, 1, 1, ghext->nghostzones);
  // Step time
  // cctkGH->cctk_time += cctkGH->cctk_delta_time;
  CCTK_VINFO("t=%g", cctkGH->cctk_time);

Replacement in WaveToyAMReX/src/wavetoy.cxx at line 123 [3.37]

B:BD[3.7339] → [3.7339:7394]

  ghext->mfab.FillBoundary(ghext->geom.periodicity());

[3.7339]

[3.7394]

  for (auto &restrict groupdata : ghext->groupdata)
    groupdata.mfab.FillBoundary(ghext->geom.periodicity());

Replacement in WaveToyAMReX/src/wavetoy.cxx at line 152 [3.37]

B:BD[3.9113] → [3.9113:9237]

B:BD[3.9237] → [2.4193:4269]

∅:D[2.4269] → [5.6615:6670]

B:BD[3.9315] → [5.6615:6670]

  string filename = amrex::Concatenate("wavetoy/phi", cctk_iteration, 6);
  WriteSingleLevelPlotfile(filename, ghext->mfab,
                           {"phi", "phi_p", "phi_p_p", "err"}, ghext->geom,
                           cctk_time, cctk_iteration);

[3.9113]

[3.9372]

  const int numgroups = CCTK_NumGroups();
  for (int gi = 0; gi < numgroups; ++gi) {
    auto &restrict groupdata = ghext->groupdata.at(gi);
    string groupname = unique_ptr<char>(CCTK_GroupName(gi)).get();
    groupname = regex_replace(groupname, regex("::"), "-");
    for (auto &c : groupname)
      c = tolower(c);
    ostringstream buf;
    buf << "wavetoy/" << groupname << "." << setw(6) << setfill('0')
        << cctk_iteration;
    string filename = buf.str();
    Vector<string> varnames(groupdata.numvars * groupdata.numtimelevels);
    for (int tl = 0; tl < groupdata.numtimelevels; ++tl) {
      for (int n = 0; n < groupdata.numvars; ++n) {
        ostringstream buf;
        buf << CCTK_VarName(groupdata.firstvarindex + n);
        for (int i = 0; i < tl; ++i)
          buf << "_p";
        varnames.at(tl * groupdata.numvars + n) = buf.str();
      }
    }
    WriteSingleLevelPlotfile(filename, groupdata.mfab, varnames, ghext->geom,
                             cctk_time, cctk_iteration);
  }

Overload evolution manager

Dependencies

In channels

Change contents

Insertion in AMReX/param.ccl at line 2 [3.17918]

Insertion in AMReX/src/AMReX.hxx at line 14 [3.15275]

Deletion in AMReX/src/AMReX.hxx at line 36 [3.15275]

Replacement in AMReX/src/AMReX.hxx at line 37 [3.15275]

Insertion in AMReX/src/driver.cxx at line 58 [3.15972]

Deletion in AMReX/src/driver.cxx at line 109 [3.15972]

Replacement in AMReX/src/driver.cxx at line 113 [3.15972]

Insertion in AMReX/src/schedule.cxx at line 8 [6.531]

Insertion in AMReX/src/schedule.cxx at line 17 [6.531]

Replacement in AMReX/src/schedule.cxx at line 128 [6.531]

Replacement in AMReX/src/schedule.cxx at line 148 [6.531]

Insertion in AMReX/src/schedule.cxx at line 176 [6.531]

Insertion in AMReX/src/schedule.cxx at line 329 [6.531]

Replacement in AMReX/src/schedule.cxx at line 333 [6.531]

Deletion in AMReX/src/schedule.cxx at line 335 [6.531]

Insertion in AMReX/src/schedule.hxx at line 16 [6.5992]

Insertion in AMReXTest/interface.ccl at line 6 [7.4807]

Replacement in AMReXTest/src/check_loops.cxx at line 36 [7.5134]

Replacement in AMReXTest/src/check_loops.cxx at line 42 [7.5134]

Replacement in AMReXTest/src/check_loops.cxx at line 71 [7.5134]

Replacement in AMReXTest/src/check_loops.cxx at line 79 [7.5134]

Replacement in AMReXTest/src/check_loops.cxx at line 81 [7.5134]

Replacement in AMReXTest/src/check_loops.cxx at line 101 [7.5134]

Replacement in AMReXTest/src/check_loops.cxx at line 107 [7.5134]

Replacement in WaveToyAMReX/interface.ccl at line 9 [3.10196]

Replacement in WaveToyAMReX/interface.ccl at line 14 [3.10196]

Deletion in WaveToyAMReX/interface.ccl at line 16 [3.10196]

Replacement in WaveToyAMReX/interface.ccl at line 17 [3.10196]

Deletion in WaveToyAMReX/schedule.ccl at line 13 [3.9594]

Insertion in WaveToyAMReX/src/wavetoy.cxx at line 13 [3.37]

Insertion in WaveToyAMReX/src/wavetoy.cxx at line 15 [3.37]

Deletion in WaveToyAMReX/src/wavetoy.cxx at line 89 [3.37]

Replacement in WaveToyAMReX/src/wavetoy.cxx at line 123 [3.37]

Replacement in WaveToyAMReX/src/wavetoy.cxx at line 152 [3.37]