2DKSL6DKZAIYQUJGDULORCKU5K4Z5Z3W4RIKQYDSLKMCNQNDZFBAC
3KQ5ZQE45TNGATMW4R4NHXVBULGOXHF7JYC53BZL2LNDQ2V7C2CAC
TOBGHRPKEPSXDN56WGSZNWOMCBVJ4KUSLWYWI56MC2RR3MM3KLZAC
722HZ7UFINNE3YKSYKP2NHZ5XEG5QQLQHSKC7PREJZR3EX6RDYUAC
MSBBCXVGD3GRLE5KAI6BKAFRV7SQUWI2SNN43AJAUD3ISRCEXY6QC
RTNZAS3UPI6GG3KY4Z5WVXJ4R2YF5427BB6WAV3GHRS5W7XPOSUQC
UUGQGVC4WEKN64WAP7F5QPS2UHGQB5ZLMFRIYNWKMIEBDO3QRX4AC
TVBD244E7Q7WV44CRBTFST535NUP3JAZH6OLL4IKDR3OWEXSU7HAC
QN2UTSQP4IMCMVXZNR24J22N24ASGXF6EWXQ2P3VWVQERUPFAFDQC
KG47IF4CPCUT3BHS34WDRHTH5HYMBTY4OSTB3X7APR2E5ZJ32IYQC
U77PE56ICORZNQW33NXGSEMW7GDHCSSZ4EXB6OHBJSHEG6WHYSSQC
BVR7DVINVPQG7PA6Z7QYVYNQ43YZL7XCC6AOMSMWMGAAB2Q43STAC
IEFVL4X5BPCLUX5D24ESP7FTFSX2HMCD2GOIFCMSWNUYEHQEJOIAC
RCLGQ2LZMFVPBPTU2G55DJ6HZPOGGTPZRZCY54VGP6YLHANJ2LAQC
OJZWEAJRLOA5FZAC6FMGBQ6XFQFDNO55IRWLFWDAYWM54MUIU5LAC
UZAKARMGORRQG733ZUPJOEGL5FG243I32NCC2SRSFDCZKUQ5A52QC
VAF66DTVLDWNG7N2PEQYEH4OH5SPSMFBXKPR2PU67IIM6CVPCJ7AC
2XYZZL42IEZHGDJA6NDKGSQKGJP24LOTLFJ6RNHOKWHHSUYIHGKQC
IVHURSHY4636OGIF3PNDO5CWOVRLJ75M4LP65J6I2E6KAM4QKF4AC
AEVGZIZEUIC52MCK3J4V547YEV2R4YQL3JUJW7FSP4R34PSZ43DAC
2DD222JSYRPHTXKSRXLSOMSCQPZUORNFLLO2P3GMIDELAAMD5MEQC
PQB3EKQ6MBCXPTW4HB7SGMSTOTYMB3EFZX2573OFCQI6PSOEKCSQC
WASO7G5FJXRXWNH2U2FLUNEKU6VE63OI3HUYP64BVD4LMD6KE7OQC
ORAAQXS3UNKXYDJBCTHNEIDJIZDHWMM5EVCZKVMFRKQK2NIQNJGAC
ULG227G2VMOOSDSKHY36QQQGEABGXZIX2B3JA4YIBS6HCVDVJ7IAC
KQNKYNRSWOY2K7M5PY362RJT4CRUJVECVSCNGKSJJBPYI3NU4GDQC
PH327KEIT3EWVB6OVULNZDHU4QUMCXBWUDPW6CCBZQJY3JFSPZIAC
} 128
} 8
CCTK_INT blocking_factor_z "Blocking factor"
{
1:* :: "must be a power of 2"
} 8
CCTK_INT max_grid_size_x "Maximum grid size"
{
1:* :: "must be a multiple of the blocking factor"
} 32
CCTK_INT max_grid_size_y "Maximum grid size"
{
1:* :: "must be a multiple of the blocking factor"
} 32
CCTK_INT max_grid_size_z "Maximum grid size"
{
1:* :: "must be a multiple of the blocking factor"
} 32
CCTK_INT max_tile_size_x "Maximum tile size"
{
1:* :: ""
} 1024000
CCTK_INT max_tile_size_y "Maximum tile size"
{
1:* :: ""
} 16
CCTK_INT max_tile_size_z "Maximum tile size"
{
1:* :: ""
} 32
SHARES: IO
USES STRING out_dir
USES CCTK_INT out_every
const Array4<CCTK_REAL> &ctagarr = groupdata.mfab.at(tl)->array(mfi);
const CCTK_REAL *restrict ctags = ctagarr.ptr(imin.x, imin.y, imin.z, vi);
const Array4<char> &atagarr = tags.array(mfi);
char *restrict atags = atagarr.ptr(imin.x, imin.y, imin.z, vi);
const Array4<CCTK_REAL> &vars = groupdata.mfab.at(tl)->array(mfi);
const CCTK_REAL *restrict ptr = vars.ptr(imin.x, imin.y, imin.z, vi);
const Array4<char> &tagarr = tags.array(mfi);
// Set blocking factors via parameter table since AmrMesh needs to
// know them when its constructor is running, but there are no
// constructor arguments for them
ParmParse pp;
pp.add("amr.blocking_factor_x", blocking_factor_x);
pp.add("amr.blocking_factor_y", blocking_factor_y);
pp.add("amr.blocking_factor_z", blocking_factor_z);
pp.add("amr.max_grid_size_x", max_grid_size_x);
pp.add("amr.max_grid_size_y", max_grid_size_y);
pp.add("amr.max_grid_size_z", max_grid_size_z);
#warning "TODO: increase blocking factor"
const int blocking_factor = 8;
// const int blocking_factor = 1;
ghext->amrcore->SetBlockingFactor(blocking_factor);
const int max_grid_size = 32;
ghext->amrcore->SetMaxGridSize(max_grid_size);
int maxnumlevels = ghext->amrcore->maxLevel() + 1;
for (int level = 0; level < maxnumlevels; ++level) {
CCTK_VINFO("Geometry level %d:", level);
cout << ghext->amrcore->Geom(level) << "\n";
if (verbose) {
int maxnumlevels = ghext->amrcore->maxLevel() + 1;
for (int level = 0; level < maxnumlevels; ++level) {
CCTK_VINFO("Geometry level %d:", level);
cout << ghext->amrcore->Geom(level) << "\n";
}
namespace {
// Convert a (direction, face) pair to an AMReX Orientation
Orientation orient(int d, int f) {
return Orientation(d, Orientation::Side(f));
}
} // namespace
template <typename T>
void write_arrays(ostream &os, const cGH *restrict cctkGH, int level,
int component, const vector<const T *> ptrs,
const int *restrict ash, const int *restrict lbnd,
const int *restrict lsh, const int *restrict bbox,
const int *restrict nghostzones) {
DECLARE_CCTK_ARGUMENTS;
const int levfac = 1 << level;
CCTK_REAL x0[dim], dx[dim];
for (int d = 0; d < dim; ++d) {
dx[d] = cctk_delta_space[d] / levfac;
x0[d] = cctk_origin_space[d] + 0.5 * dx[d];
}
int imin[dim], imax[dim];
for (int d = 0; d < dim; ++d) {
imin[d] = cctk_bbox[2 * d] ? 0 : cctk_nghostzones[d];
imax[d] = cctk_lsh[d] - (cctk_bbox[2 * d + 1] ? 0 : cctk_nghostzones[d]);
}
constexpr int di = 1;
const int dj = di * ash[0];
const int dk = dj * ash[1];
for (int k = 0; k < lsh[2]; ++k) {
for (int j = 0; j < lsh[1]; ++j) {
for (int i = 0; i < lsh[0]; ++i) {
const int idx = di * i + dj * j + dk * k;
const int isghost = !(i >= imin[0] && i < imax[0] && j >= imin[1] &&
j < imax[1] && k >= imin[2] && k < imax[2]);
T x = x0[0] + (lbnd[0] + i) * dx[0];
T y = x0[1] + (lbnd[1] + j) * dx[1];
T z = x0[2] + (lbnd[2] + k) * dx[2];
os << cctk_iteration << "\t" << cctk_time << "\t" << level << "\t"
<< component << "\t" << isghost << "\t" << (lbnd[0] + i) << "\t"
<< (lbnd[1] + j) << "\t" << (lbnd[2] + k) << "\t" << x << "\t" << y
<< "\t" << z;
for (const auto &ptr : ptrs)
os << "\t" << ptr[idx];
os << "\n";
}
}
}
}
void WriteASCII(const cGH *restrict cctkGH, const string &filename, int gi,
const vector<string> &varnames) {
ostringstream buf;
buf << filename << ".tsv";
ofstream file(buf.str());
// Output header
file << "#"
<< "\t"
<< "time"
<< "\t"
<< "level"
<< "\t"
<< "component"
<< "\t"
<< "isghost"
<< "\t"
<< "i"
<< "\t"
<< "j"
<< "\t"
<< "k"
<< "\t"
<< "x"
<< "\t"
<< "y"
<< "\t"
<< "z";
for (const auto &varname : varnames)
file << "\t" << varname;
file << "\n";
for (const auto &leveldata : ghext->leveldata) {
const auto &groupdata = leveldata.groupdata.at(gi);
const int tl = 0;
const MultiFab &mfab = *groupdata.mfab.at(tl);
for (MFIter mfi(mfab); mfi.isValid(); ++mfi) {
const Box &fbx = mfi.fabbox(); // allocated array
const Box &bx = mfi.tilebox(); // current region (without ghosts)
const Box &gbx = mfi.growntilebox(); // current region (with ghosts)
const Dim3 imin = lbound(gbx);
// Local shape
int lsh[dim];
for (int d = 0; d < dim; ++d)
lsh[d] = gbx[orient(d, 1)] - gbx[orient(d, 0)] + 1;
// Allocated shape
int ash[dim];
for (int d = 0; d < dim; ++d)
ash[d] = fbx[orient(d, 1)] - fbx[orient(d, 0)] + 1;
// Local extent
int lbnd[dim];
for (int d = 0; d < dim; ++d)
lbnd[d] = gbx[orient(d, 0)];
// Boundaries
int bbox[2 * dim];
for (int d = 0; d < dim; ++d)
for (int f = 0; f < 2; ++f)
bbox[2 * d + f] = bx[orient(d, f)] == gbx[orient(d, f)];
// Number of ghost zones
int nghostzones[dim];
for (int d = 0; d < dim; ++d)
nghostzones[d] = mfab.fb_nghost[d];
const Array4<CCTK_REAL> &vars = groupdata.mfab.at(tl)->array(mfi);
vector<const CCTK_REAL *> ptrs(groupdata.numvars);
for (int vi = 0; vi < groupdata.numvars; ++vi)
ptrs.at(vi) = vars.ptr(imin.x, imin.y, imin.z, vi);
write_arrays(file, cctkGH, leveldata.level, mfi.index(), ptrs, ash, lbnd,
lsh, bbox, nghostzones);
}
}
file.close();
}
// CCTK_REAL mindx = 1.0 / 0.0;
// const int numlevels = ghext->amrcore->finestLevel() + 1;
// for (int level = 0; level < numlevels; ++level) {
// const Geometry &geom = ghext->amrcore->Geom(level);
// const CCTK_REAL *restrict dx = geom.CellSize();
// for (int d = 0; d < dim; ++d)
// mindx = fmin(mindx, dx[d]);
// }
const int numlevels = ghext->amrcore->finestLevel() + 1;
for (int level = 0; level < numlevels; ++level) {
const Geometry &geom = ghext->amrcore->Geom(level);
const CCTK_REAL *restrict dx = geom.CellSize();
for (int d = 0; d < dim; ++d)
mindx = fmin(mindx, dx[d]);
}
for (int d = 0; d < dim; ++d)
mindx = fmin(mindx, dx[d]);
mindx = mindx / (1 << (max_num_levels - 1));
CCTK_VINFO("CallFunction [%d] %s: %s::%s", cctkGH->cctk_iteration,
attribute->where, attribute->thorn, attribute->routine);
if (verbose)
CCTK_VINFO("CallFunction [%d] %s: %s::%s", cctkGH->cctk_iteration,
attribute->where, attribute->thorn, attribute->routine);
for (int tl = 0; tl < sync_tl; ++tl)
InterpFromCoarseLevel(
*groupdata.mfab.at(tl), 0.0, *coarsegroupdata.mfab.at(tl), 0, 0,
groupdata.numvars, ghext->amrcore->Geom(level - 1),
ghext->amrcore->Geom(level), cphysbc, 0, fphysbc, 0, reffact,
&interp, bcs, 0);
for (int tl = 0; tl < sync_tl; ++tl) {
// InterpFromCoarseLevel(
// *groupdata.mfab.at(tl), 0.0, *coarsegroupdata.mfab.at(tl), 0,
// 0, groupdata.numvars, ghext->amrcore->Geom(level - 1),
// ghext->amrcore->Geom(level), cphysbc, 0, fphysbc, 0, reffact,
// &interp, bcs, 0);
#warning "TODO: make copy of fine level"
FillPatchTwoLevels(
*groupdata.mfab.at(tl), 0.0, {&*coarsegroupdata.mfab.at(tl)},
{0.0}, {&*groupdata.mfab.at(tl)}, {0.0}, 0, 0, groupdata.numvars,
ghext->amrcore->Geom(level - 1), ghext->amrcore->Geom(level),
cphysbc, 0, fphysbc, 0, reffact, &cell_bilinear_interp, bcs, 0);
}
2. First-order equations:
Scalar wave equation:
phi,tt = phi,xx
Introduce new variable:
psi = phi,t
New set of equations:
phi,t = psi
psi,t = phi,tt = phi,xx
Discretization:
phi_0 = phi_1 + dt psi_1
psi_0 = psi_1 + dt phi_1,xx
3. Fancy equations:
Scalar wave equation:
phi,tt = phi,xx
Direct second order discretization:
phi_0 = 2 phi_1 - phi_2 + dt^2 phi_1,xx
Introduce new variable:
psi = phi,t
New set of equations:
phi,t = psi
psi,t = phi,tt = phi,xx
New, equivalent discretization:
phi_0 - phi_1 = dt psi_1
phi_0 - 2 phi_1 + phi_2 = dt^2 phi_1,xx
dt psi_1 - dt psi_2 = dt^2 phi_1,xx
psi_1 = psi_2 + dt^2 phi_1,xx
phi_0 = phi_1 + dt psi_1
psi_0 = psi_1 + dt phi_0,xx
= psi_1 + dt phi_1,xx + dt^2 psi_1,xx
for (int k = 0; k < cctk_lsh[2]; ++k) {
for (int j = 0; j < cctk_lsh[1]; ++j) {
int imin[dim], imax[dim];
for (int d = 0; d < dim; ++d) {
imin[d] = cctk_bbox[2 * d] ? 0 : cctk_nghostzones[d];
imax[d] = cctk_lsh[d] - (cctk_bbox[2 * d + 1] ? 0 : cctk_nghostzones[d]);
}
for (int k = imin[2]; k < imax[2]; ++k) {
for (int j = imin[1]; j < imax[1]; ++j) {
phi[idx] = -phi_p_p[idx] + 2 * phi_p[idx] +
// CCTK_REAL ddx_psi =
// (psi_p[idx - di] - 2 * psi_p[idx] + psi_p[idx + di]) /
// sqr(dx[0]);
// CCTK_REAL ddy_psi =
// (psi_p[idx - dj] - 2 * psi_p[idx] + psi_p[idx + dj]) /
// sqr(dx[1]);
// CCTK_REAL ddz_psi =
// (psi_p[idx - dk] - 2 * psi_p[idx] + psi_p[idx + dk]) /
// sqr(dx[2]);
// phi[idx] = phi_p[idx] + dt * psi_p[idx];
// psi[idx] = psi_p[idx] + dt * (ddx_phi + ddy_phi + ddz_phi) +
// sqr(dt) * (ddx_psi + ddy_psi + ddz_psi);
// phi[idx] = phi_p[idx] + dt * psi_p[idx];
// psi[idx] = psi_p[idx] + dt * (ddx_phi + ddy_phi + ddz_phi);
phi[idx] = 2 * phi_p[idx] - phi_p_p[idx] +
}
}
}
}
extern "C" void WaveToyAMReX_Energy(CCTK_ARGUMENTS) {
DECLARE_CCTK_ARGUMENTS;
DECLARE_CCTK_PARAMETERS;
const CCTK_REAL dt = CCTK_DELTA_TIME;
CCTK_REAL dx[dim];
for (int d = 0; d < dim; ++d)
dx[d] = CCTK_DELTA_SPACE(d);
int imin[dim], imax[dim];
for (int d = 0; d < dim; ++d) {
imin[d] = cctk_bbox[2 * d] ? 0 : cctk_nghostzones[d];
imax[d] = cctk_lsh[d] - (cctk_bbox[2 * d + 1] ? 0 : cctk_nghostzones[d]);
}
constexpr int di = 1;
const int dj = di * cctk_ash[0];
const int dk = dj * cctk_ash[1];
for (int k = imin[2]; k < imax[2]; ++k) {
for (int j = imin[1]; j < imax[1]; ++j) {
#pragma omp simd
for (int i = imin[0]; i < imax[0]; ++i) {
const int idx = CCTK_GFINDEX3D(cctkGH, i, j, k);
CCTK_REAL dt_phi = (phi[idx] - phi_p[idx]) / dt;
CCTK_REAL dx_phi = (phi[idx + di] - phi[idx - di]) / (2 * dx[0]);
CCTK_REAL dy_phi = (phi[idx + dj] - phi[idx - dj]) / (2 * dx[1]);
CCTK_REAL dz_phi = (phi[idx + dk] - phi[idx - dk]) / (2 * dx[2]);
eps[idx] = (sqr(dt_phi) + sqr(dx_phi) + sqr(dy_phi) + sqr(dz_phi)) / 2;
ActiveThorns = "
AMReX
IOUtil
WaveToyAMReX
"
$maxlev = 1
$ncells = 128
Cactus::cctk_show_schedule = no
Cactus::cctk_itlast = $ncells * 2 ** ($maxlev - 1) * 2
AMReX::ncells_x = $ncells
AMReX::ncells_y = 1
AMReX::ncells_z = 1
AMReX::blocking_factor_x = 1
AMReX::blocking_factor_y = 1
AMReX::blocking_factor_z = 1
AMReX::max_grid_size_x = 1048576
AMReX::max_grid_size_y = 1048576
AMReX::max_grid_size_z = 1048576
AMReX::max_num_levels = $maxlev
AMReX::dtfac = 0.5 / 4
IO::out_dir = "wavetoy-1"
IO::out_every = $ncells * 2 ** ($maxlev - 1) / 2
ActiveThorns = "
AMReX
IOUtil
WaveToyAMReX
"
$maxlev = 2
$ncells = 128
Cactus::cctk_show_schedule = no
Cactus::cctk_itlast = $ncells * 2 ** ($maxlev - 1) * 2
AMReX::ncells_x = $ncells
AMReX::ncells_y = 1
AMReX::ncells_z = 1
AMReX::blocking_factor_x = 1
AMReX::blocking_factor_y = 1
AMReX::blocking_factor_z = 1
AMReX::max_grid_size_x = 1048576
AMReX::max_grid_size_y = 1048576
AMReX::max_grid_size_z = 1048576
AMReX::max_num_levels = $maxlev
AMReX::dtfac = 0.5 / 2
IO::out_dir = "wavetoy-2"
IO::out_every = $ncells * 2 ** ($maxlev - 1) / 2
ActiveThorns = "
AMReX
IOUtil
WaveToyAMReX
"
$maxlev = 3
$ncells = 128
Cactus::cctk_show_schedule = no
Cactus::cctk_itlast = $ncells * 2 ** ($maxlev - 1) * 2
AMReX::ncells_x = $ncells
AMReX::ncells_y = 1
AMReX::ncells_z = 1
AMReX::blocking_factor_x = 1
AMReX::blocking_factor_y = 1
AMReX::blocking_factor_z = 1
AMReX::max_grid_size_x = 1048576
AMReX::max_grid_size_y = 1048576
AMReX::max_grid_size_z = 1048576
AMReX::max_num_levels = $maxlev
AMReX::dtfac = 0.5
IO::out_dir = "wavetoy-3"
IO::out_every = $ncells * 2 ** ($maxlev - 1) / 2