# Component list for the Einstein Toolkit <http://einsteintoolkit.org/>
# $Revision$
# $Date$
# $HeadURL$
!CRL_VERSION = 1.0
!DEFINE ROOT = Cactus
!DEFINE ARR  = $ROOT/arrangements
!DEFINE COMPONENTLIST_TARGET = $ROOT/thornlists
# To download these thorns via GetComponents, use then the following
# command (on the same system where you originally used
# GetComponents):
#   cd Cactus (or whatever your source tree is called)
#   bin/GetComponents --update --root=. manifest/einsteintoolkit.th
# This thorn list
!TARGET   = $ROOT
!TYPE     = git
!URL      = https://bitbucket.org/einsteintoolkit/manifest.git
!REPO_PATH= $1
!NAME     = manifest
!CHECKOUT = ./manifest
# Cactus Flesh
!TARGET   = $ROOT
!TYPE     = git
!URL      = https://bitbucket.org/cactuscode/cactus.git
!NAME     = flesh
!CHECKOUT = .clang-format CONTRIBUTORS COPYRIGHT doc lib Makefile src
# Simulation Factory
!TARGET   = $ROOT
!TYPE     = git
!URL      = https://bitbucket.org/simfactory/simfactory2.git
!REPO_PATH=$1
!NAME     = simfactory2
!CHECKOUT = ./simfactory
# Example parameter files
!TARGET   = $ROOT
!TYPE     = git
!URL      = https://bitbucket.org/einsteintoolkit/einsteinexamples.git
!CHECKOUT = par
# Various Cactus utilities
!TARGET   = $ROOT
!TYPE     = git
!URL      = https://bitbucket.org/cactuscode/utilities.git
!REPO_PATH= $1
!NAME     = utils
!CHECKOUT = ./utils
# The GetComponents script
!TARGET   = $ROOT/bin
!TYPE     = git
!URL      = https://github.com/gridaphobe/CRL.git
!CHECKOUT = GetComponents
# CactusBase thorns
!TARGET   = $ARR
!TYPE     = git
!URL      = https://bitbucket.org/cactuscode/cactusbase.git
!REPO_PATH= $2
!CHECKOUT =
CactusBase/Boundary
# CactusBase/CartGrid3D
CactusBase/CoordBase
CactusBase/Fortran
CactusBase/InitBase
# CactusBase/IOASCII
CactusBase/IOBasic
CactusBase/IOUtil
CactusBase/SymBase
CactusBase/Time
# Additional Cactus thorns
!TARGET   = $ARR
!TYPE     = svn
!URL      = https://svn.cactuscode.org/projects/$1/$2/trunk
!CHECKOUT =
ExternalLibraries/MPI
# AMReX thorns
AMReX/AMReX
# AMReX/AMReXTest
AMReX/WaveToyAMReX

    export AMREX_DIR=$HOME/amrex

- script: |
    wget https://raw.githubusercontent.com/gridaphobe/CRL/ET_2019_03/GetComponents
    chmod a+x GetComponents
    ./GetComponents --parallel $HOME/cactusamrex/amrex.th
    cd Cactus
    echo 'ubuntu >$HOME/.hostname
    cp cactusamrex/defs.local.ini simfactory/etc/defs.local.ini
    cp cactusamrex/ubuntu.ini simfactory/mdb/machines/ubuntu.ini
    cp cactusamrex/ubuntu.cfg simfactory/mdb/optionlists/ubuntu.cfg
    cp cactusamrex/ubuntu.sub simfactory/mdb/submitscripts/ubuntu.sub
    cp cactusamrex/ubuntu.run simfactory/mdb/runscripts/ubuntu.run
    ./simfactory/bin/sim build -j2 --thornlist=$HOME/cactusamrex/amrex.th
  displayName: Build Cactus

# -*-conf-*-
# Defaults for all machines
[default]
user            = cactus
email           = cactus@localhost
thornlist       = manifest/einsteintoolkit.th
[ubuntu]
user            = cactus

# Configuration for an Ubuntu installation, assuming the following
# list of packages is installed:
#
#    build-essential perl python gfortran g++ libmpich-dev
#
# In addition, installing the following list of packaed will prevent
# Cactus from compiling its own versions of these libraries:
#
#   libfftw3-dev libgsl-dev libatlas-base-dev libjpeg-dev libssl-dev 
#   libhdf5-dev hdf5-tools libnuma-dev libltdl-dev libhwloc-dev zlib1g-dev
#
# Tools like GetComponents and Simfactory like to have the following list
# installed too
#
#   python subversion git
# Whenever this version string changes, the application is configured 
# and rebuilt from scratch 
VERSION = ubuntu-2019-06-07
CPP = cpp
FPP = cpp
CC  = gcc
CXX = g++
F77 = gfortran
F90 = gfortran
CPPFLAGS = -DMPICH_IGNORE_CXX_SEEK
FPPFLAGS = -traditional
CFLAGS   = -pipe -g3 -march=ARCH -std=gnu11
CXXFLAGS = -pipe -g3 -march=ARCH -std=gnu++14
F77FLAGS = -pipe -g3 -march=ARCH -fcray-pointer -ffixed-line-length-none -fno-range-check
F90FLAGS = -pipe -g3 -march=ARCH -fcray-pointer -ffixed-line-length-none -fno-range-check
LDFLAGS  = -rdynamic
LIBDIRS =
C_LINE_DIRECTIVES = yes
F_LINE_DIRECTIVES = yes
DEBUG           = no
CPP_DEBUG_FLAGS = -DCARPET_DEBUG
FPP_DEBUG_FLAGS = -DCARPET_DEBUG
C_DEBUG_FLAGS   = -O0 
CXX_DEBUG_FLAGS = -O0 
F77_DEBUG_FLAGS = -O0
F90_DEBUG_FLAGS = -O0
OPTIMISE           = yes
CPP_OPTIMISE_FLAGS = -DKRANC_VECTORS # -DCARPET_OPTIMISE -DNDEBUG
FPP_OPTIMISE_FLAGS = # -DCARPET_OPTIMISE -DNDEBUG
C_OPTIMISE_FLAGS   = -O2 -ffast-math -fno-finite-math-only
CXX_OPTIMISE_FLAGS = -O2 -ffast-math -fno-finite-math-only
F77_OPTIMISE_FLAGS = -O2 -ffast-math -fno-finite-math-only
F90_OPTIMISE_FLAGS = -O2 -ffast-math -fno-finite-math-only
PROFILE           = no
CPP_PROFILE_FLAGS =
FPP_PROFILE_FLAGS =
C_PROFILE_FLAGS   = -pg
CXX_PROFILE_FLAGS = -pg
F77_PROFILE_FLAGS = -pg
F90_PROFILE_FLAGS = -pg
OPENMP           = yes
CPP_OPENMP_FLAGS = -fopenmp
FPP_OPENMP_FLAGS = -fopenmp
C_OPENMP_FLAGS   = -fopenmp
CXX_OPENMP_FLAGS = -fopenmp
F77_OPENMP_FLAGS = -fopenmp
F90_OPENMP_FLAGS = -fopenmp
WARN           = yes
CPP_WARN_FLAGS = -Wall
FPP_WARN_FLAGS = -Wall
C_WARN_FLAGS   = -Wall
CXX_WARN_FLAGS = -Wall
F77_WARN_FLAGS = -Wall
F90_WARN_FLAGS = -Wall
# Ubuntu's libhwloc-dev is broken; see
# <https://trac.einsteintoolkit.org/ticket/1717>.
# You need to add the libnuma-dev libltdl-dev packages to use the
# system hwloc package and prevent this bug.
# On the other hand you must NOT install libnuma-dev unless you also
# install libhwloc-dev and libtldl-dev.
BLAS_DIR = /usr
LAPACK_DIR = /usr
MPI_DIR = /usr
OPENBLAS_DIR = /usr
PTHREADS_DIR = NO_BUILD

[docker]
# last-tested-on: 2019-06-06
# last-tested-by: Erik Schnetter <schnetter@gmail.com>
# NOTE: presubmission is not supported
# Machine description
nickname        = docker
name            = Docker
location        = anywhere
description     = Running in a Docker container
status          = personal
# Access to this machine
hostname        = docker
aliaspattern    = ^docker$
# Source tree management
sourcebasedir   = /cactus
disabled-thorns = <<EOT
    ExternalLibraries/BLAS
    ExternalLibraries/LAPACK
EOT
enabled-thorns = <<EOT
    CactusTest/TestAllTypes
    #TODO Carpet/CarpetSimulationIO
    ExternalLibraries/OpenBLAS
    #TODO ExternalLibraries/SimulationIO
    #TODO McLachlan/ML_BSSN_ND
    #TODO McLachlan/ML_BSSN_ND_Helper
    #TODO McLachlan_dgtile/ML_BSSN_FH_SD4
EOT
optionlist      = ubuntu.cfg
submitscript    = docker.sub
runscript       = docker.run
make            = make -j @MAKEJOBS@
makejobs        = 2
# Simulation management
basedir         = /simulations
cpu             = Intel Core i7
cpufreq         = 2.6     # unknown
flop/cycle      = 16      # unknown
hyperthreads    = 2       # unknown
ppn             = 6       # unknown
spn             = 1       # unknown
mpn             = 1       # unknown
max-num-threads = 12      # unknown
num-threads     = 6       # unknown
memory          = 32768   # unknown
nodes           = 1       # unknown
min-ppn         = 1             # don't want this entry
queue           = NOQUEUE       # don't want this entry
maxwalltime     = 8640:00:00    # don't want this entry
# submit          = exec @SCRIPTFILE@ </dev/null >/dev/null 2>/dev/null & echo $!
submit          = @SCRIPTFILE@ </dev/null >/dev/null 2>/dev/null; true & echo $!
getstatus       = ps @JOB_ID@
stop            = kill @JOB_ID@
submitpattern   = (.*)
statuspattern   = "^ *@JOB_ID@ "
queuedpattern   = $^
runningpattern  = ^
holdingpattern  = $^
exechost        = echo localhost
exechostpattern = (.*)
stdout          = cat @SIMULATION_NAME@.out
stderr          = cat @SIMULATION_NAME@.err
stdout-follow   = tail -n 100 -f @SIMULATION_NAME@.out @SIMULATION_NAME@.err

#! /bin/bash
echo "Preparing:"
set -x                          # Output commands
set -e                          # Abort on errors
cd @RUNDIR@-active
echo "Checking:"
pwd
hostname
date
echo "Environment:"
export 'SIMULATION_ID=@SIMULATION_ID@'
export CACTUS_NUM_PROCS=@NUM_PROCS@
export CACTUS_NUM_THREADS=@NUM_THREADS@
export CACTUS_SET_THREAD_BINDINGS=0
export CXX_MAX_TASKS=1000
export GLIBCXX_FORCE_NEW=1
export GMON_OUT_PREFIX=gmon.out
export OMP_DISPLAY_ENV=TRUE
export OMP_MAX_TASKS=100
export OMP_NUM_THREADS=@NUM_THREADS@
export OMP_PLACES=cores     # threads, cores, sockets
export OMP_PROC_BIND=FALSE  # false, true, master, close, spread
export OMP_STACKSIZE=8192   # kByte
env | sort >SIMFACTORY/ENVIRONMENT
# Enable this to allow core file
# ulimit -c unlimited
echo "Starting:"
# Disabling TCP BTL
export CACTUS_STARTTIME=$(date +%s)
time                                            \
    /usr/bin/mpirun                             \
    -np @NUM_PROCS@                             \
    --mca btl self,vader                        \
    -x CACTUS_NUM_PROCS                         \
    -x CACTUS_NUM_THREADS                       \
    -x CACTUS_SET_THREAD_BINDINGS               \
    -x CACTUS_STARTTIME                         \
    -x CXX_MAX_TASKS                            \
    -x GLIBCXX_FORCE_NEW                        \
    -x GMON_OUT_PREFIX                          \
    -x OMP_DISPLAY_ENV                          \
    -x OMP_MAX_TASKS                            \
    -x OMP_NUM_THREADS                          \
    -x OMP_PLACES                               \
    -x OMP_PROC_BIND                            \
    -x OMP_STACKSIZE                            \
    @EXECUTABLE@ -L 3 @PARFILE@                 \
    >@SIMULATION_NAME@.out                      \
    2>@SIMULATION_NAME@.err
echo "Stopping:"
date
echo "Done."

#! /bin/bash
cd @SOURCEDIR@
exec @SIMFACTORY@ run @SIMULATION_NAME@ --machine=@MACHINE@ --restart-id=@RESTART_ID@ @FROM_RESTART_COMMAND@ </dev/null >@RUNDIR@/@SIMULATION_NAME@.log 2>&1