eschnett/cactusamrex - Change 5B6D6UREX556XOAM75B3WOIGE6DT43HIS6JOVJYNXUT3FSZDUPBAC

CarpetX: Added support for arrays and DynamicData This is specifically for distrib=const arrays, and distributed arrays are still not supported. The overloaded function for DynamicData is also included which reproduces the behavior of the Carpet DynamicData function. Finally, a bug with validity for scalars is fixed.

Created by Samuel Cupp on October 9, 2020

5B6D6UREX556XOAM75B3WOIGE6DT43HIS6JOVJYNXUT3FSZDUPBAC

Dependencies

In channels

main

Change contents

Insertion in CarpetX/src/driver.cxx at line 62 [8.15972]

[6.166]

[7.191]

int GroupDynamicData(const cGH *cctkGH, int gi, cGroupDynamicData *data);

Insertion in CarpetX/src/driver.cxx at line 632 [8.15972]
[2.3320]
[2.3320]
```
  globaldata.arraygroupdata.resize(numgroups);
```

Replacement in CarpetX/src/driver.cxx at line 639 [8.15972]

B:BD[2.3504] → [2.3504:3578]

    if (group.grouptype != CCTK_SCALAR and group.grouptype != CCTK_ARRAY)

[2.3504]

[2.3578]

    if (group.grouptype != CCTK_SCALAR and group.grouptype != CCTK_ARRAY) {

Insertion in CarpetX/src/driver.cxx at line 641 [8.15972]

[2.3594]

    } else if (group.grouptype == CCTK_SCALAR) {
      assert(group.vartype == CCTK_VARIABLE_REAL);
      assert(group.disttype == CCTK_DISTRIB_CONSTANT);
      assert(group.dim == 0);
      globaldata.scalargroupdata.at(gi) =
          make_unique<GHExt::GlobalData::ScalarGroupData>();
      GHExt::GlobalData::ScalarGroupData &scalargroupdata =
          *globaldata.scalargroupdata.at(gi);
      scalargroupdata.groupindex = gi;
      scalargroupdata.firstvarindex = CCTK_FirstVarIndexI(gi);
      scalargroupdata.numvars = group.numvars;
      scalargroupdata.do_checkpoint = get_group_checkpoint_flag(gi);
      scalargroupdata.do_restrict = get_group_restrict_flag(gi);

Replacement in CarpetX/src/driver.cxx at line 656 [8.15972]

B:BD[2.3595] → [2.3595:3769]

    assert(group.grouptype == CCTK_SCALAR);
    assert(group.vartype == CCTK_VARIABLE_REAL);
    assert(group.disttype == CCTK_DISTRIB_CONSTANT);
    assert(group.dim == 0);

[2.3595]

[2.3769]

      // Set up dynamic data
      scalargroupdata.dimension = 0;
      scalargroupdata.activetimelevels = 1;
      // Allocate data
      scalargroupdata.data.resize(group.numtimelevels);
      scalargroupdata.valid.resize(group.numtimelevels);
      for (int tl = 0; tl < int(scalargroupdata.data.size()); ++tl) {
        scalargroupdata.data.at(tl).resize(scalargroupdata.numvars);
        why_valid_t why([] { return "SetupGlobals"; });
        scalargroupdata.valid.at(tl).resize(scalargroupdata.numvars, why);
        for (int vi = 0; vi < scalargroupdata.numvars; ++vi) {
          // TODO: decide that valid_bnd == false always and rely on
          // initialization magic?
          valid_t valid;
          valid.valid_int = false;
          valid.valid_outer = true;
          valid.valid_ghosts = true;
          scalargroupdata.valid.at(tl).at(vi).set(valid,
                                                  [] { return "SetupGlobals"; });
          // TODO: make poison_invalid and check_invalid virtual members of
          // CommonGroupData
          poison_invalid(scalargroupdata, vi, tl);
          check_valid(scalargroupdata, vi, tl, [] { return "SetupGlobals"; });
        }
      }
    } else {
      assert(group.grouptype == CCTK_ARRAY);
      assert(group.vartype == CCTK_VARIABLE_REAL);
      assert(group.disttype == CCTK_DISTRIB_CONSTANT);
      assert(group.dim > 0 && group.dim <= dim);
      globaldata.arraygroupdata.at(gi) =
          make_unique<GHExt::GlobalData::ArrayGroupData>();
      GHExt::GlobalData::ArrayGroupData &arraygroupdata =
          *globaldata.arraygroupdata.at(gi);
      arraygroupdata.groupindex = gi;
      arraygroupdata.firstvarindex = CCTK_FirstVarIndexI(gi);
      arraygroupdata.numvars = group.numvars;
      arraygroupdata.do_checkpoint = get_group_checkpoint_flag(gi);
      arraygroupdata.do_restrict = get_group_restrict_flag(gi);

Replacement in CarpetX/src/driver.cxx at line 699 [8.15972]

B:BD[2.3770] → [2.3770:4244]

    globaldata.scalargroupdata.at(gi) =
        make_unique<GHExt::GlobalData::ScalarGroupData>();
    GHExt::GlobalData::ScalarGroupData &scalargroupdata =
        *globaldata.scalargroupdata.at(gi);
    scalargroupdata.groupindex = gi;
    scalargroupdata.firstvarindex = CCTK_FirstVarIndexI(gi);
    scalargroupdata.numvars = group.numvars;
    scalargroupdata.do_checkpoint = get_group_checkpoint_flag(gi);
    scalargroupdata.do_restrict = get_group_restrict_flag(gi);

[2.3770]

[2.4244]

      CCTK_INT const *const *const sz = CCTK_GroupSizesI(gi);
      arraygroupdata.array_size = 1;
      for (int d = 0; d < group.dim; ++d) {
        arraygroupdata.array_size = arraygroupdata.array_size * *sz[d];
      }

Replacement in CarpetX/src/driver.cxx at line 705 [8.15972]

B:BD[2.4245] → [2.4245:5058]

    // Allocate data
    scalargroupdata.data.resize(group.numtimelevels);
    scalargroupdata.valid.resize(group.numtimelevels);
    for (int tl = 0; tl < int(scalargroupdata.data.size()); ++tl) {
      scalargroupdata.data.at(tl).resize(scalargroupdata.numvars);
      why_valid_t why([] { return "SetupGlobals"; });
      scalargroupdata.valid.at(tl).resize(scalargroupdata.numvars, why);
      for (int vi = 0; vi < scalargroupdata.numvars; ++vi) {
        // TODO: decide that valid_bnd == false always and rely on
        // initialization magic?
        valid_t valid;
        valid.valid_int = false;
        valid.valid_outer = true;
        valid.valid_ghosts = true;
        scalargroupdata.valid.at(tl).at(vi).set(valid,
                                                [] { return "SetupGlobals"; });

[2.4245]

[2.5058]

      // Set up dynamic data
      arraygroupdata.dimension = group.dim;
      arraygroupdata.activetimelevels = 1;
      for (int d = 0; d < group.dim; ++d) {
        arraygroupdata.lsh[d] = *sz[d];
        arraygroupdata.ash[d] = *sz[d];
        arraygroupdata.gsh[d] = *sz[d];
        arraygroupdata.nghostzones[d] = 0;
        arraygroupdata.lbnd[d] = 0;
        arraygroupdata.ubnd[d] = *sz[d]-1;
        arraygroupdata.bbox[2*d] = arraygroupdata.bbox[2*d+1] = 1;
      }
      // Allocate data
      arraygroupdata.data.resize(group.numtimelevels);
      arraygroupdata.valid.resize(group.numtimelevels);
      for (int tl = 0; tl < int(arraygroupdata.data.size()); ++tl) {
        arraygroupdata.data.at(tl).resize(arraygroupdata.numvars*arraygroupdata.array_size);
        why_valid_t why([] { return "SetupGlobals"; });
        arraygroupdata.valid.at(tl).resize(arraygroupdata.numvars, why);
        for (int vi = 0; vi < arraygroupdata.numvars; ++vi) {
          // TODO: decide that valid_bnd == false always and rely on
          // initialization magic?
          valid_t valid;
          valid.valid_int = false;
          valid.valid_outer = true;
          valid.valid_ghosts = true;
          arraygroupdata.valid.at(tl).at(vi).set(valid,
                                                  [] { return "SetupGlobals"; });

Replacement in CarpetX/src/driver.cxx at line 735 [8.15972]

B:BD[2.5059] → [2.5059:5286]

        // TODO: make poison_invalid and check_invalid virtual members of
        // CommonGroupData
        poison_invalid(scalargroupdata, vi, tl);
        check_valid(scalargroupdata, vi, tl, [] { return "SetupGlobals"; });

[2.5059]

[2.5286]

          // TODO: make poison_invalid and check_invalid virtual members of
          // CommonGroupData
          poison_invalid(arraygroupdata, vi, tl);
          check_valid(arraygroupdata, vi, tl, [] { return "SetupGlobals"; });
        }

Insertion in CarpetX/src/driver.cxx at line 1180 [8.15972]

[4.1238]

YAML::Emitter &
operator<<(YAML::Emitter &yaml,
           const GHExt::GlobalData::ArrayGroupData &arraygroupdata) {
  yaml << YAML::LocalTag("arraygroupdata-1.0.0");
  yaml << YAML::BeginMap;
  yaml << YAML::Key << "commongroupdata" << YAML::Value
       << (GHExt::CommonGroupData)arraygroupdata;
  yaml << YAML::Key << "data" << YAML::Value << arraygroupdata.data;
  yaml << YAML::EndMap;
  return yaml;
}

Insertion in CarpetX/src/driver.cxx at line 1385 [8.15972]
[9.888]
[9.888]
```
  CCTK_OverloadGroupDynamicData(GroupDynamicData);
```

Insertion in CarpetX/src/driver.cxx at line 1650 [8.15972]

[5.2153]

[10.2206]

  }
  return 0;
}
int GroupDynamicData(const cGH *cctkGH, int gi, cGroupDynamicData *data) {
  // Return values:
  //  0 for success
  // -1 if given pointer to data structure is NULL
  // -3 if given GH pointer is invalid
  // (-77 if group has zero variables)
  // -78 if group does not exist
  if (not cctkGH)
    return -3;
  if (not(gi >= 0 and gi < CCTK_NumGroups()))
    return -78;
  if (not data)
    return -1;
  cGroup group;
  int ierr = CCTK_GroupData(gi, &group);
  assert(!ierr);
  if (group.grouptype != CCTK_SCALAR and group.grouptype != CCTK_ARRAY) {
    data->dim = group.dim;
    data->lsh = cctkGH->cctk_lsh;
    data->ash = cctkGH->cctk_ash;
    data->gsh = cctkGH->cctk_gsh;
    data->lbnd = cctkGH->cctk_lbnd;
    data->ubnd = cctkGH->cctk_ubnd;
    data->bbox = cctkGH->cctk_bbox;
    data->nghostzones = cctkGH->cctk_nghostzones;
    data->activetimelevels = CCTK_ActiveTimeLevelsGI(cctkGH, gi);
  } else if (group.grouptype == CCTK_SCALAR) {
    GHExt::GlobalData &globaldata = ghext->globaldata;
    GHExt::GlobalData::ScalarGroupData &scalargroupdata =
        *globaldata.scalargroupdata.at(gi);
    data->dim = scalargroupdata.dimension;
    data->lsh = scalargroupdata.lsh;
    data->ash = scalargroupdata.ash;
    data->gsh = scalargroupdata.gsh;
    data->lbnd = scalargroupdata.lbnd;
    data->ubnd = scalargroupdata.ubnd;
    data->bbox = scalargroupdata.bbox;
    data->nghostzones = scalargroupdata.nghostzones;
    data->activetimelevels = scalargroupdata.activetimelevels;
  } else { // CCTK_ARRAY
    GHExt::GlobalData &globaldata = ghext->globaldata;
    GHExt::GlobalData::ArrayGroupData &arraygroupdata =
        *globaldata.arraygroupdata.at(gi);
    data->dim = arraygroupdata.dimension;
    data->lsh = arraygroupdata.lsh;
    data->ash = arraygroupdata.ash;
    data->gsh = arraygroupdata.gsh;
    data->lbnd = arraygroupdata.lbnd;
    data->ubnd = arraygroupdata.ubnd;
    data->bbox = arraygroupdata.bbox;
    data->nghostzones = arraygroupdata.nghostzones;
    data->activetimelevels = arraygroupdata.activetimelevels;

Replacement in CarpetX/src/driver.hxx at line 114 [8.15275]

B:BD[11.1774] → [11.1774:1845]

    // integrated grid scalar per level. We don't do that yet; insted,

[11.1774]

[11.1845]

    // integrated grid scalar per level. We don't do that yet; instead,

Insertion in CarpetX/src/driver.hxx at line 119 [8.15275]

[12.66]

[4.4468]

      int dimension;
      int activetimelevels;
      int lsh[dim];
      int ash[dim];
      int gsh[dim];
      int nghostzones[dim];
      int bbox[2*dim];
      int lbnd[dim];
      int ubnd[dim];

Insertion in CarpetX/src/driver.hxx at line 129 [8.15275]

[4.4469]

      ScalarGroupData() {
        dimension = -1;
        activetimelevels = -1;
        for (int d = 0; d < dim; d++) {
          lsh[d] = -1;
          ash[d] = -1;
          gsh[d] = -1;
          lbnd[d] = -1;
          ubnd[d] = -1;
          bbox[2*d] = bbox[2*d+1] = -1;
          nghostzones[d] = -1;
        }
      }

Insertion in CarpetX/src/driver.hxx at line 148 [8.15275]

[13.763]

[4.4610]


    struct ArrayGroupData : public CommonGroupData {
      vector<vector<CCTK_REAL> > data; // [time level][var index]
      int array_size;
      int dimension;
      int activetimelevels;
      int lsh[dim];
      int ash[dim];
      int gsh[dim];
      int lbnd[dim];
      int ubnd[dim];
      int bbox[2*dim];
      int nghostzones[dim];

Insertion in CarpetX/src/driver.hxx at line 162 [8.15275]

[4.4611]

      ArrayGroupData() {
        dimension = -1;
        activetimelevels = -1;
        for (int d = 0; d < dim; d++) {
          lsh[d] = -1;
          ash[d] = -1;
          gsh[d] = -1;
          lbnd[d] = -1;
          ubnd[d] = -1;
          bbox[2*d] = bbox[2*d+1] = -1;
          nghostzones[d] = -1;
        }
      }
      friend YAML::Emitter &operator<<(YAML::Emitter &yaml,
                                       const ArrayGroupData &arraygroupdata);
    };
    // TODO: right now this is sized for the total number of groups
    vector<unique_ptr<ArrayGroupData> > arraygroupdata; // [group index]

Replacement in CarpetX/src/schedule.cxx at line 483 [15.531]
B:BD[14.6907] → [14.6907:6942]
```
// Ensure grid functions are valid
```
[14.6907]
[16.2683]
```
// Ensure grid scalars are valid
```

Replacement in CarpetX/src/schedule.cxx at line 489 [15.531]

B:BD[14.7228] → [14.7228:7301]

    CCTK_VERROR("%s: Grid function \"%s\" is invalid on time level %d; "

[14.7228]

[14.7301]

    CCTK_VERROR("%s: Grid Scalar \"%s\" is invalid on time level %d; "

Replacement in CarpetX/src/schedule.cxx at line 501 [15.531]

B:BD[14.7849] → [14.7849:7921]

               "%s: Grid function \"%s\" is invalid on time level %d; "

[14.7849]

[14.7921]

               "%s: Grid Scalar \"%s\" is invalid on time level %d; "

Deletion in CarpetX/src/schedule.cxx at line 518 [15.531]
B:BD[17.5941] → [17.5941:5942]
B:BD[17.5942] → [19.0:66]
B:BD[19.66] → [20.8993:9044]
```
  // scalars have no boundary so we expect them to alway be valid
  assert(valid.valid_outer && valid.valid_ghosts);
```

Insertion in CarpetX/src/schedule.cxx at line 555 [15.531]

[21.4885]

[22.2]

}
// Ensure grid arrays are valid
void error_if_invalid(const GHExt::GlobalData::ArrayGroupData &groupdata,
                      int vi, int tl, const valid_t &required,
                      const function<string()> &msg) {
  const valid_t &have = groupdata.valid.at(tl).at(vi).get();
  if (CCTK_BUILTIN_EXPECT((required & ~have).valid_any(), false))
    CCTK_VERROR("%s: Grid array \"%s\" is invalid on time level %d; "
                "required %s, found %s",
                msg().c_str(), CCTK_FullVarName(groupdata.firstvarindex + vi),
                tl, string(required).c_str(),
                string(groupdata.valid.at(tl).at(vi)).c_str());
}
void warn_if_invalid(const GHExt::GlobalData::ArrayGroupData &groupdata,
                     int vi, int tl, const valid_t &required,
                     const function<string()> &msg) {
  const valid_t &have = groupdata.valid.at(tl).at(vi).get();
  if (CCTK_BUILTIN_EXPECT((required & ~have).valid_any(), false))
    CCTK_VWARN(CCTK_WARN_ALERT,
               "%s: Grid array \"%s\" is invalid on time level %d; "
               "required %s, found %s",
               msg().c_str(), CCTK_FullVarName(groupdata.firstvarindex + vi),
               tl, string(required).c_str(),
               string(groupdata.valid.at(tl).at(vi)).c_str());
}
// Set (distrib=const) grid arrays to nan
void poison_invalid(const GHExt::GlobalData::ArrayGroupData &arraygroupdata,
                    int vi, int tl) {
  DECLARE_CCTK_PARAMETERS;
  if (!poison_undefined_values)
    return;
  const valid_t &valid = arraygroupdata.valid.at(tl).at(vi).get();
  if (valid.valid_all())
    return;
//TODO: poison needs to poison entire array, not just the first element
  if (!valid.valid_int) {
    CCTK_REAL *restrict const ptr =
        const_cast<CCTK_REAL *>(&arraygroupdata.data.at(tl).at(vi));
    *ptr = 0.0 / 0.0;
  }
}
// Ensure grid arrays are not nan
void check_valid(const GHExt::GlobalData::ArrayGroupData &arraygroupdata,
                 int vi, int tl, const function<string()> &msg) {
  DECLARE_CCTK_PARAMETERS;
  if (!poison_undefined_values)
    return;
  const valid_t &valid = arraygroupdata.valid.at(tl).at(vi).get();
  if (!valid.valid_any())
    return;
  // arrays have no boundary so we expect them to alway be valid
  assert(valid.valid_outer && valid.valid_ghosts);
//TODO: nan checker needs to check every element of the arrays
//  int ubnd = *arraygroupdata.info->ubnd;
  atomic<size_t> nan_count{0};
  if (valid.valid_int) {
    const CCTK_REAL *restrict const ptr = &arraygroupdata.data.at(tl).at(vi);
//    for (int i = 0; i < ubnd; i++) {
      if (CCTK_BUILTIN_EXPECT(!CCTK_isfinite(*ptr), false)) {
        ++nan_count;
      }
//    }
  }
  if (CCTK_BUILTIN_EXPECT(nan_count > 0, false))
    CCTK_VERROR("%s: Grid Array \"%s\" has %td nans on time level %d; "
                "expected valid %s",
                msg().c_str(),
                CCTK_FullVarName(arraygroupdata.firstvarindex + vi),
                size_t(nan_count), tl,
                string(arraygroupdata.valid.at(tl).at(vi)).c_str());

Replacement in CarpetX/src/schedule.cxx at line 974 [15.531]

B:BD[17.7480] → [17.7480:7522]

      if (group.grouptype != CCTK_SCALAR)

[17.7480]

[17.7522]

      if (group.grouptype != CCTK_SCALAR && group.grouptype != CCTK_ARRAY) {

Replacement in CarpetX/src/schedule.cxx at line 976 [15.531]

B:BD[17.7540] → [17.7540:7541]

B:BD[17.7541] → [13.1635:1710]

∅:D[13.1710] → [17.7615:7813]

B:BD[17.7615] → [17.7615:7813]

B:BD[17.7813] → [12.313:431]


      auto &restrict scalargroupdata = *globaldata.scalargroupdata.at(gi);
      for (int tl = 0; tl < int(scalargroupdata.data.size()); ++tl) {
        const auto &restrict vars = scalargroupdata.data.at(tl);
        for (int vi = 0; vi < scalargroupdata.numvars; ++vi) {
          cctkGH->data[scalargroupdata.firstvarindex + vi][tl] =
              const_cast<CCTK_REAL *>(&vars.at(vi));

[17.7540]

[17.7891]

      } else if (group.grouptype == CCTK_SCALAR) {
        auto &restrict scalargroupdata = *globaldata.scalargroupdata.at(gi);
        for (int tl = 0; tl < int(scalargroupdata.data.size()); ++tl) {
          const auto &restrict vars = scalargroupdata.data.at(tl);
          for (int vi = 0; vi < scalargroupdata.numvars; ++vi) {
            cctkGH->data[scalargroupdata.firstvarindex + vi][tl] =
                const_cast<CCTK_REAL *>(&vars.at(vi));
          }

Insertion in CarpetX/src/schedule.cxx at line 985 [15.531]

[17.7901]

      } else { // CCTK_ARRAY
        auto &restrict arraygroupdata = *globaldata.arraygroupdata.at(gi);
        for (int tl = 0; tl < int(arraygroupdata.data.size()); ++tl) {
          const auto &restrict vars = arraygroupdata.data.at(tl);
          for (int vi = 0; vi < arraygroupdata.numvars; ++vi) {
            cctkGH->data[arraygroupdata.firstvarindex + vi][tl] =
                const_cast<CCTK_REAL *>(&vars.at(vi*arraygroupdata.array_size));
          }
        }

Replacement in CarpetX/src/schedule.cxx at line 1010 [15.531]

B:BD[17.8179] → [17.8179:8221]

      if (group.grouptype != CCTK_SCALAR)

[17.8179]

[17.8221]

      if (group.grouptype != CCTK_SCALAR && group.grouptype != CCTK_ARRAY) {

Replacement in CarpetX/src/schedule.cxx at line 1012 [15.531]

B:BD[17.8239] → [17.8239:8240]

B:BD[17.8240] → [13.1711:1786]

∅:D[13.1786] → [17.8314:8521]

B:BD[17.8314] → [17.8314:8521]


      auto &restrict scalargroupdata = *globaldata.scalargroupdata.at(gi);
      for (int tl = 0; tl < int(scalargroupdata.data.size()); ++tl) {
        for (int vi = 0; vi < scalargroupdata.numvars; ++vi) {
          cctkGH->data[scalargroupdata.firstvarindex + vi][tl] = nullptr;

[17.8239]

[17.8521]

      } else if (group.grouptype == CCTK_SCALAR) {
        auto &restrict scalargroupdata = *globaldata.scalargroupdata.at(gi);
        for (int tl = 0; tl < int(scalargroupdata.data.size()); ++tl) {
          for (int vi = 0; vi < scalargroupdata.numvars; ++vi) {
            cctkGH->data[scalargroupdata.firstvarindex + vi][tl] = nullptr;
          }
        }
      } else { // CCTK_ARRAY
        auto &restrict arraygroupdata = *globaldata.arraygroupdata.at(gi);
        for (int tl = 0; tl < int(arraygroupdata.data.size()); ++tl) {
          for (int vi = 0; vi < arraygroupdata.numvars; ++vi) {
            cctkGH->data[arraygroupdata.firstvarindex + vi][tl] = nullptr;
          }

Replacement in CarpetX/src/schedule.cxx at line 1625 [15.531]
∅:D[3.1981] → [23.6157:6163]
B:BD[23.6157] → [23.6157:6163]
B:BD[17.9667] → [17.9667:9668]
B:BD[17.9668] → [24.8501:8569]
```
    }
    // Invalidate scalars
    if (group.grouptype == CCTK_SCALAR) {
```
[3.1981]
[17.9728]
```
    } else if (group.grouptype == CCTK_SCALAR) {
```

Insertion in CarpetX/src/schedule.cxx at line 1640 [15.531]

[17.10228]

          }
        }
      }
    } else { // CCTK_ARRAY
      auto &restrict globaldata = ghext->globaldata;
      auto &restrict arraygroupdata = *globaldata.arraygroupdata.at(gi);
      if (!arraygroupdata.do_checkpoint) {
        // Invalidate all time levels
        const int ntls = arraygroupdata.data.size();
        for (int tl = 0; tl < ntls; ++tl) {
          for (int vi = 0; vi < arraygroupdata.numvars; ++vi) {
            // TODO: handle this more nicely
            arraygroupdata.valid.at(tl).at(vi).set_int(false, [] {
              return "InvalidateTimelevels (invalidate all non-checkpointed "
                     "variables)";
            });
            poison_invalid(arraygroupdata, vi, tl);

Replacement in CarpetX/src/schedule.cxx at line 1701 [15.531]
∅:D[25.2071] → [26.3462:3468]
∅:D[3.2080] → [26.3462:3468]
∅:D[27.13087] → [26.3462:3468]
B:BD[26.3462] → [26.3462:3468]
```
    }
```
[3.2080]
[17.10818]
```
    } else if (group.grouptype == CCTK_SCALAR) {
```
Deletion in CarpetX/src/schedule.cxx at line 1703 [15.531]
B:BD[17.10819] → [24.9905:9968]
∅:D[24.9968] → [17.10879:10880]
B:BD[17.10879] → [17.10879:10880]
```
    // cycle scalars
    if (group.grouptype == CCTK_SCALAR) {
```

Insertion in CarpetX/src/schedule.cxx at line 1722 [15.531]

[24.10246]

                      [&]() { return "CycleTimelevels"; });
    } else { // CCTK_ARRAY
      auto &restrict globaldata = ghext->globaldata;
      auto &restrict arraygroupdata = *globaldata.arraygroupdata.at(gi);
      const int ntls = arraygroupdata.data.size();
      // Rotate time levels and invalidate current time level
      if (ntls > 1) {
        rotate(arraygroupdata.data.begin(), arraygroupdata.data.end() - 1,
               arraygroupdata.data.end());
        rotate(arraygroupdata.valid.begin(), arraygroupdata.valid.end() - 1,
               arraygroupdata.valid.end());
        for (int vi = 0; vi < arraygroupdata.numvars; ++vi) {
          arraygroupdata.valid.at(0).at(vi).set_int(false, [] {
            return "CycletimeLevels (invalidate current time level)";
          });
          poison_invalid(arraygroupdata, vi, 0);
        }
      }
      for (int tl = 0; tl < ntls; ++tl)
        for (int vi = 0; vi < arraygroupdata.numvars; ++vi)
          check_valid(arraygroupdata, vi, tl,

Replacement in CarpetX/src/schedule.cxx at line 1944 [15.531]
B:BD[28.1540] → [28.1540:1570]
```
      } else { // CCTK_SCALAR
```
[28.1540]
[28.1570]
```
      } else if (CCTK_GroupTypeI(rd.gi) == CCTK_SCALAR) {
```

Insertion in CarpetX/src/schedule.cxx at line 1957 [15.531]

[14.10196]

[28.2263]

          ostringstream buf;
          buf << "CallFunction iteration " << cctkGH->cctk_iteration << " "
              << attribute->where << ": " << attribute->thorn
              << "::" << attribute->routine << " checking input";
          return buf.str();
        });
      } else { // CCTK_ARRAY
        const auto &restrict arraygroupdata =
            *ghext->globaldata.arraygroupdata.at(rd.gi);
        const valid_t &need = rd.valid;
        error_if_invalid(arraygroupdata, rd.vi, rd.tl, need, [&] {
          ostringstream buf;
          buf << "CallFunction iteration " << cctkGH->cctk_iteration << " "
              << attribute->where << ": " << attribute->thorn
              << "::" << attribute->routine << " checking input";
          return buf.str();
        });
        check_valid(arraygroupdata, rd.vi, rd.tl, [&] {

Replacement in CarpetX/src/schedule.cxx at line 2022 [15.531]
B:BD[28.3100] → [28.3100:3130]
```
      } else { // CCTK_SCALAR
```
[28.3100]
[17.12893]
```
      } else if (CCTK_GroupTypeI(wr.gi) == CCTK_SCALAR) {
```

Insertion in CarpetX/src/schedule.cxx at line 2038 [15.531]

[28.3396]

[29.44]


      } else { // CCTK_ARRAY
        auto &restrict arraygroupdata =
            *ghext->globaldata.arraygroupdata.at(wr.gi);
        const valid_t &provided = wr.valid;
        arraygroupdata.valid.at(wr.tl).at(wr.vi).set_and(
            need | ~provided,
            [iteration = cctkGH->cctk_iteration, where = attribute->where,
             thorn = attribute->thorn, routine = attribute->routine] {
              ostringstream buf;
              buf << "CallFunction iteration " << iteration << " " << where
                  << ": " << thorn << "::" << routine
                  << ": Poison output variables that are not input variables";
              return buf.str();
            });
        poison_invalid(arraygroupdata, wr.vi, wr.tl);

Replacement in CarpetX/src/schedule.cxx at line 2219 [15.531]
B:BD[28.4212] → [28.4212:4242]
```
      } else { // CCTK_SCALAR
```
[28.4212]
[17.13348]
```
      } else if (CCTK_GroupTypeI(wr.gi) == CCTK_SCALAR) {
```

Insertion in CarpetX/src/schedule.cxx at line 2235 [15.531]

[28.4469]

          ostringstream buf;
          buf << "CallFunction iteration " << cctkGH->cctk_iteration << " "
              << attribute->where << ": " << attribute->thorn
              << "::" << attribute->routine << " checking output";
          return buf.str();
        });
      } else { // CCTK_ARRAY
        auto &restrict arraygroupdata =
            *ghext->globaldata.arraygroupdata.at(wr.gi);
        const valid_t &provided = wr.valid;
        arraygroupdata.valid.at(wr.tl).at(wr.vi).set_or(
            provided,
            [iteration = cctkGH->cctk_iteration, where = attribute->where,
             thorn = attribute->thorn, routine = attribute->routine] {
              ostringstream buf;
              buf << "CallFunction iteration " << iteration << " " << where
                  << ": " << thorn << "::" << routine
                  << ": Mark output variables as valid";
              return buf.str();
            });
        check_valid(arraygroupdata, wr.vi, wr.tl, [&]() {

Replacement in CarpetX/src/schedule.cxx at line 2297 [15.531]
B:BD[28.5570] → [28.5570:5600]
```
      } else { // CCTK_SCALAR
```
[28.5570]
[18.2166]
```
      } else if (CCTK_GroupTypeI(inv.gi) == CCTK_SCALAR) {
```

Insertion in CarpetX/src/schedule.cxx at line 2313 [15.531]

[28.5832]

          ostringstream buf;
          buf << "CallFunction iteration " << cctkGH->cctk_iteration << " "
              << attribute->where << ": " << attribute->thorn
              << "::" << attribute->routine << " checking output";
          return buf.str();
        });
      } else { // CCTK_ARRAY
        auto &restrict arraygroupdata =
            *ghext->globaldata.arraygroupdata.at(inv.gi);
        const valid_t &provided = inv.valid;
        arraygroupdata.valid.at(inv.tl).at(inv.vi).set_and(
            ~provided,
            [iteration = cctkGH->cctk_iteration, where = attribute->where,
             thorn = attribute->thorn, routine = attribute->routine] {
              ostringstream buf;
              buf << "CallFunction iteration " << iteration << " " << where
                  << ": " << thorn << "::" << routine
                  << ": Mark invalid variables as invalid";
              return buf.str();
            });
        check_valid(arraygroupdata, inv.vi, inv.tl, [&]() {

Insertion in CarpetX/src/schedule.hxx at line 179 [15.5992]

[17.14511]

[30.4359]

void error_if_invalid(const GHExt::GlobalData::ArrayGroupData &groupdata,
                      int vi, int tl, const valid_t &required,
                      const function<string()> &msg);
void warn_if_invalid(const GHExt::GlobalData::ArrayGroupData &groupdata,
                     int vi, int tl, const valid_t &required,
                     const function<string()> &msg);
void poison_invalid(const GHExt::GlobalData::ArrayGroupData &groupdata, int vi,
                    int tl);
void check_valid(const GHExt::GlobalData::ArrayGroupData &groupdata, int vi,
                 int tl, const function<string()> &msg);