ORNL AF cluster 800-node AMD Epyc 2-sockets 64-cores per node CNL gnu,cray,intel mpt nwp501 /lustre/storm/nwp501/proj-shared/e3sm e3sm,nwp501 /lustre/storm/nwp501/proj-shared/e3sm/e3sm_scratch/ /lustre/storm/nwp501/proj-shared/e3sm/inputdata /lustre/storm/nwp501/proj-shared/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /lustre/storm/nwp501/proj-shared/e3sm/baselines/$COMPILER /lustre/storm/nwp501/proj-shared/e3sm/tools/cprnc 8 e3sm_developer 4 miller_slurm e3sm 128 128 FALSE srun -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit -c $SHELL{echo 128/ {{ tasks_per_node }} |bc} $SHELL{if [ 128 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -m plane={{ tasks_per_node }} /opt/cray/pe/modules/default/init/perl.pm /opt/cray/pe/modules/default/init/python.py /opt/cray/pe/modules/default/init/sh /opt/cray/pe/modules/default/init/csh /opt/cray/pe/modules/default/bin/modulecmd perl /opt/cray/pe/modules/default/bin/modulecmd python module module craype cray-mpich cray-parallel-netcdf cray-hdf5-parallel cray-hdf5 cray-netcdf cray-netcdf-hdf5parallel craype/2.7.15 PrgEnv-cray PrgEnv-gnu PrgEnv-gnu/8.3.3 gcc/12.1.0 PrgEnv-gnu gcc/9.3.0 PrgEnv-cray/8.3.3 darshan cray-mpich/8.1.16 cray-hdf5-parallel/1.12.1.3 cray-netcdf-hdf5parallel/4.8.1.3 cray-parallel-netcdf/1.12.2.3 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1000 /usr/lib/perl5/5.26.2 /opt/cray/pe/netcdf-hdf5parallel/4.8.1.3/gnu/9.1/ /opt/cray/pe/netcdf-hdf5parallel/4.8.1.3/gnu/9.1/ $SHELL{dirname $(dirname $(which pnetcdf_version))} 128M cores Perlmutter CPU-only nodes at NERSC. Phase2 only: Each node has 2 AMD EPYC 7713 64-Core (Milan) 512GB $ENV{NERSC_HOST}:perlmutter Linux gnu,intel,nvidia,amdclang mpich e3sm /global/cfs/cdirs/e3sm e3sm,m3411,m3412 $ENV{PSCRATCH}/e3sm_scratch/pm-cpu /global/cfs/cdirs/e3sm/www/$ENV{USER} http://portal.nersc.gov/project/e3sm/$ENV{USER} /global/cfs/cdirs/e3sm/inputdata /global/cfs/cdirs/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /global/cfs/cdirs/e3sm/baselines/$COMPILER /global/cfs/cdirs/e3sm/tools/cprnc.cori/cprnc 10 e3sm_developer 4 nersc_slurm e3sm 256 128 TRUE srun --label -n {{ total_tasks }} -N {{ num_nodes }} -c $SHELL{echo 256/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} $SHELL{if [ 128 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -m plane=$SHELL{echo `./xmlquery --value MAX_MPITASKS_PER_NODE`} /usr/share/lmod/8.3.1/init/perl /usr/share/lmod/8.3.1/init/python /usr/share/lmod/8.3.1/init/sh /usr/share/lmod/8.3.1/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module module cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf PrgEnv-gnu PrgEnv-intel PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc intel intel-oneapi cudatoolkit craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan PrgEnv-gnu/8.3.3 gcc/11.2.0 cray-libsci/23.02.1.1 PrgEnv-intel/8.3.3 intel/2023.1.0 PrgEnv-nvidia nvidia/22.7 cray-libsci/23.02.1.1 PrgEnv-aocc aocc/3.2.0 cray-libsci/23.02.1.1 craype-accel-host craype/2.7.20 cray-mpich/8.1.25 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 cmake/3.24.3 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 0.20 1 1 128M spread threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch software MPI_Bcast /global/cfs/cdirs/e3sm/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.15/gcc-11.2.0 /global/cfs/cdirs/e3sm/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.15/nvidia-21.11 /global/cfs/cdirs/e3sm/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.15/aocc-3.2.0 -1 Perlmutter GPU nodes at NERSC. Phase1 only: Each GPU node has single AMD EPYC 7713 64-Core (Milan) (256GB) and 4 nvidia A100's. $ENV{NERSC_HOST}:perlmutter Linux gnugpu,gnu,nvidiagpu,nvidia mpich e3sm_g /global/cfs/cdirs/e3sm e3sm,m3411,m3412 $ENV{PSCRATCH}/e3sm_scratch/pm-gpu /global/cfs/cdirs/e3sm/www/$ENV{USER} http://portal.nersc.gov/project/e3sm/$ENV{USER} /global/cfs/cdirs/e3sm/inputdata /global/cfs/cdirs/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /global/cfs/cdirs/e3sm/baselines/$COMPILER /global/cfs/cdirs/e3sm/tools/cprnc.cori/cprnc 10 e3sm_developer 4 nersc_slurm e3sm 128 256 256 4 64 64 TRUE srun --label -n {{ total_tasks }} -N {{ num_nodes }} -c $SHELL{echo 128/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} $SHELL{if [ 64 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -m plane=$SHELL{echo `./xmlquery --value MAX_MPITASKS_PER_NODE`} /usr/share/lmod/8.3.1/init/perl /usr/share/lmod/8.3.1/init/python /usr/share/lmod/8.3.1/init/sh /usr/share/lmod/8.3.1/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module module cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf PrgEnv-gnu PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc cudatoolkit craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan PrgEnv-gnu/8.3.3 gcc/11.2.0 PrgEnv-nvidia nvidia/22.7 cudatoolkit/11.5 craype-accel-nvidia80 cudatoolkit/11.5 craype-accel-nvidia80 craype-accel-host craype-accel-host cray-libsci/23.02.1.1 craype/2.7.20 cray-mpich/8.1.25 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 cmake/3.24.3 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1 1 128M spread threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch MPI_Bcast 1 1 /global/cfs/cdirs/e3sm/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.15/gcc-11.2.0 /global/cfs/cdirs/e3sm/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.15/nvidia-21.11 -1 test machine at NERSC, very similar to pm-cpu. each node has 2 AMD EPYC 7713 64-Core (Milan) 512GB $ENV{NERSC_HOST}:alvarez Linux gnu,nvidia,amdclang mpich e3sm /global/cfs/cdirs/e3sm e3sm,m3411,m3412 $ENV{PSCRATCH}/e3sm_scratch/alvarez /global/cfs/cdirs/e3sm/www/$ENV{USER} http://portal.nersc.gov/project/e3sm/$ENV{USER} /global/cfs/cdirs/e3sm/inputdata /global/cfs/cdirs/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /global/cfs/cdirs/e3sm/baselines/$COMPILER /global/cfs/cdirs/e3sm/tools/cprnc.cori/cprnc 10 e3sm_developer 4 nersc_slurm e3sm 256 128 TRUE srun --label -n {{ total_tasks }} -N {{ num_nodes }} -c $SHELL{echo 256/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} $SHELL{if [ 128 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -m plane=$SHELL{echo `./xmlquery --value MAX_MPITASKS_PER_NODE`} /usr/share/lmod/8.3.1/init/perl /usr/share/lmod/8.3.1/init/python /usr/share/lmod/8.3.1/init/sh /usr/share/lmod/8.3.1/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module module cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf PrgEnv-gnu PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc cudatoolkit craype-accel-nvidia80 craype-accel-host perftools-base perftools darshan PrgEnv-gnu/8.3.3 gcc/12.1.0 PrgEnv-nvidia nvidia/22.7 PrgEnv-aocc aocc/3.2.0 craype-accel-host cray-libsci/23.02.1.1 craype/2.7.19 cray-mpich/8.1.24 cray-hdf5-parallel/1.12.2.3 cray-netcdf-hdf5parallel/4.9.0.3 cray-parallel-netcdf/1.12.3.3 cmake/3.24.3 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1 1 128M spread threads FALSE /global/cfs/cdirs/e3sm/perl/lib/perl5-only-switch software MPI_Bcast -1 Spock. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. .*spock.* Linux gnu,cray mpich cli133 /gpfs/alpine/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch/spock /gpfs/alpine/cli115/world-shared/e3sm/inputdata /gpfs/alpine/cli115/world-shared/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /gpfs/alpine/cli133/world-shared/grnydawn/e3sm/tools/cprnc_spock/cprnc 8 1 slurm e3sm 64 64 TRUE srun --label -l -K -n {{ total_tasks }} -N {{ num_nodes }} --cpu_bind=cores -c $ENV{OMP_NUM_THREADS} -m plane={{ tasks_per_node }} /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/libexec/lmod perl module module /usr/share/lmod/lmod/libexec/lmod python DefApps cray-python/3.8.5.1 subversion/1.14.0 git/2.31.1 cmake/3.20.2 zlib/1.2.11 cray-libsci/21.06.1.1 PrgEnv-gnu/8.0.0 cray-mpich/8.1.7 cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.5 PrgEnv-cray/8.0.0 cray-mpich/8.1.7 cray-hdf5-parallel/1.12.0.6 cray-netcdf-hdf5parallel/4.7.4.6 cray-parallel-netcdf/1.12.1.5 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which pnetcdf_version))} 128M spread threads Frontier exascale supercomputer at ORNL. 9408 nodes, Node: 4 AMD MI250X GPUs (2 GCDs) ~ 8 GPUs, 512 GB HDB2E, AMD EPYC 64 cores, 512GB DDR4 .*frontier.* Linux crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu mpich cli115 /lustre/orion/cli115/world-shared/frontier .* /lustre/orion/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch /lustre/orion/cli115/world-shared/e3sm/inputdata /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /lustre/orion/cli115/world-shared/e3sm/baselines/frontier/$COMPILER /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc 8 1 slurm e3sm 56 56 8 8 8 TRUE srun -l -K -n {{ total_tasks }} -N {{ num_nodes }} -c $ENV{OMP_NUM_THREADS} $ENV{NTASKS_PER_GPU} $ENV{GPU_BIND_ARGS} /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/libexec/lmod perl module module /usr/share/lmod/lmod/libexec/lmod python PrgEnv-cray PrgEnv-cray/8.3.3 cce cce/15.0.1 craype craype/2.7.20 craype-accel-amd-gfx90a rocm/5.4.0 PrgEnv-cray PrgEnv-amd/8.3.3 amd amd/5.4.0 craype-accel-amd-gfx90a PrgEnv-cray PrgEnv-gnu/8.3.3 gcc gcc/11.2.0 craype-accel-amd-gfx90a rocm/5.4.0 cray-python/3.9.13.1 subversion/1.14.1 git/2.36.1 cmake/3.21.3 zlib/1.2.11 cray-hdf5-parallel/1.12.2.1 cray-netcdf-hdf5parallel/4.9.0.1 cray-parallel-netcdf/1.12.3.1 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 0.25 0 $ENV{NETCDF_DIR} $ENV{PNETCDF_DIR} $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} --ntasks-per-gpu=$SHELL{echo "`./xmlquery --value MAX_MPITASKS_PER_NODE`/8"|bc} --gpu-bind=closest romio_cb_read=disable 0 10 3 128M spread threads /lustre/orion/cli133/world-shared/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.17/gcc-11.2.0 /lustre/orion/cli133/world-shared/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.17/crayclang-14.0.2 /lustre/orion/cli133/world-shared/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.17/amdclang-15.0.0 Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs .*crusher.* Linux crayclang,gnu,amdclang,gnugpu,crayclanggpu,amdclanggpu mpich cli115 /lustre/orion/cli115/world-shared/crusher .* /lustre/orion/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch/crusher /lustre/orion/cli115/world-shared/e3sm/inputdata /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /lustre/orion/cli115/world-shared/e3sm/baselines/crusher/$COMPILER /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc 8 1 slurm e3sm 56 56 8 8 8 TRUE srun -l -K -n {{ total_tasks }} -N {{ num_nodes }} -c $ENV{OMP_NUM_THREADS} $ENV{NTASKS_PER_GPU} $ENV{GPU_BIND_ARGS} /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/libexec/lmod perl module module /usr/share/lmod/lmod/libexec/lmod python PrgEnv-cray PrgEnv-cray/8.3.3 cce cce/15.0.1 craype craype/2.7.20 craype-accel-amd-gfx90a rocm/5.4.0 PrgEnv-cray PrgEnv-amd/8.3.3 amd amd/5.4.0 craype-accel-amd-gfx90a PrgEnv-cray PrgEnv-gnu/8.3.3 gcc gcc/11.2.0 craype-accel-amd-gfx90a rocm/5.4.0 cray-python/3.9.13.1 subversion/1.14.1 git/2.36.1 cmake/3.21.3 zlib/1.2.11 cray-hdf5-parallel/1.12.2.1 cray-netcdf-hdf5parallel/4.9.0.1 cray-parallel-netcdf/1.12.3.1 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 0.25 0 $ENV{NETCDF_DIR} $ENV{PNETCDF_DIR} $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} --ntasks-per-gpu=$SHELL{echo "`./xmlquery --value MAX_MPITASKS_PER_NODE`/8"|bc} --gpu-bind=closest romio_cb_read=disable 0 10 3 128M spread threads /lustre/orion/cli133/world-shared/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.17/gcc-11.2.0 /lustre/orion/cli133/world-shared/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.17/crayclang-14.0.2 /lustre/orion/cli133/world-shared/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.17/amdclang-15.0.0 Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs .*crusher.* Linux crayclang-scream mpich CLI133_crusher /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher /lustre/orion/cli115/world-shared/e3sm/inputdata /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc 8 1 slurm e3sm 56 56 TRUE srun -l -K -n {{ total_tasks }} -N {{ num_nodes }} --threads-per-core=1 -c $ENV{OMP_NUM_THREADS} -m *:block /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/libexec/lmod perl module module /usr/share/lmod/lmod/libexec/lmod python PrgEnv-cray PrgEnv-cray/8.3.3 cce cce/14.0.0 PrgEnv-cray PrgEnv-amd/8.3.3 amd amd/5.1.0 PrgEnv-cray PrgEnv-gnu/8.3.3 cray-mpich/8.1.12 cray-python/3.9.4.2 subversion/1.14.0 git/2.31.1 cmake/3.21.3 zlib/1.2.11 cray-libsci/21.08.1.2 cray-hdf5-parallel/1.12.1.1 cray-netcdf-hdf5parallel/4.8.1.1 cray-parallel-netcdf/1.12.1.7 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 $ENV{NETCDF_DIR} $ENV{PNETCDF_DIR} $ENV{CRAY_LIBSCI_DIR}/amd/4.0/x86_64/lib:$ENV{LD_LIBRARY_PATH} 0 128M spread threads /lustre/orion/cli133/world-shared/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.17/crayclang-14.0.2 Crusher. NCCS moderate-security system that contains similar hardware and software as the upcoming Frontier system at ORNL. 192 AMD EPYC 7A53 64C nodes, 128 hwthreads, 512GB DDR4, 4 MI250X GPUs .*crusher.* Linux crayclang-scream mpich CLI133_crusher /lustre/orion/cli133/proj-shared/$ENV{USER}/e3sm_scratch/crusher /lustre/orion/cli115/world-shared/e3sm/inputdata /lustre/orion/cli115/world-shared/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /lustre/orion/cli115/world-shared/e3sm/tools/cprnc/cprnc 8 1 slurm e3sm 64 8 TRUE srun -l -K -n {{ total_tasks }} -N {{ num_nodes }} --gpus-per-node=8 --gpu-bind=closest -c $ENV{OMP_NUM_THREADS} /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/libexec/lmod perl module module /usr/share/lmod/lmod/libexec/lmod python PrgEnv-cray craype-accel-amd-gfx90a rocm/5.1.0 cce/14.0.3 cray-python/3.9.4.2 subversion/1.14.0 git/2.31.1 cmake/3.21.3 zlib/1.2.11 cray-hdf5-parallel/1.12.2.1 cray-netcdf-hdf5parallel/4.9.0.1 cray-parallel-netcdf/1.12.3.1 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 0 $ENV{NETCDF_DIR} $ENV{PNETCDF_DIR} 0 1 romio_cb_read=disable 128M spread threads /lustre/orion/cli133/world-shared/3rdparty/adios2/2.8.3.patch/cray-mpich-8.1.17/crayclang-14.0.2 Stampede2. Intel skylake nodes at TACC. 48 cores per node, batch system is SLURM .*stampede2.* LINUX intel,gnu impi $ENV{SCRATCH} acme $ENV{SCRATCH}/acme_scratch/stampede2 $ENV{SCRATCH}/inputdata $ENV{SCRATCH}/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE $ENV{SCRATCH}/baselines/$COMPILER $ENV{SCRATCH}/tools/cprnc.cori/cprnc 8 e3sm_developer slurm e3sm 96 48 FALSE ibrun /opt/apps/lmod/lmod/init/perl /opt/apps/lmod/lmod/init/python /opt/apps/lmod/lmod/init/sh /opt/apps/lmod/lmod/init/csh /opt/apps/lmod/lmod/libexec/lmod perl /opt/apps/lmod/lmod/libexec/lmod python module -q module -q intel/18.0.0 gcc/6.3.0 impi/18.0.0 hdf5/1.8.16 netcdf/4.3.3.1 phdf5/1.8.16 parallel-netcdf/4.3.3.1 pnetcdf/1.8.1 git cmake autotools xalt $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1 1 128M spread threads 1 -l Mac OS/X workstation or laptop Darwin gnu openmpi,mpich $ENV{HOME}/projects/acme/scratch $ENV{HOME}/projects/acme/cesm-inputdata $ENV{HOME}/projects/acme/ptclm-data $ENV{HOME}/projects/acme/scratch/archive/$CASE $ENV{HOME}/projects/acme/baselines/$COMPILER $CCSMROOT/tools/cprnc/build/cprnc 4 e3sm_developer none jnjohnson at lbl dot gov 4 2 mpirun $ENV{HOME}/projects/acme/scratch/$CASE/run $ENV{HOME}/projects/acme/scratch/$CASE/bld Linux workstation or laptop none LINUX gnu openmpi,mpich $ENV{HOME}/projects/acme/scratch $ENV{HOME}/projects/acme/cesm-inputdata $ENV{HOME}/projects/acme/ptclm-data $ENV{HOME}/projects/acme/scratch/archive/$CASE $ENV{HOME}/projects/acme/baselines/$COMPILER $CCSMROOT/tools/cprnc/build/cprnc 4 e3sm_developer none jayesh at mcs dot anl dot gov 4 2 mpirun -np {{ total_tasks }} $ENV{HOME}/projects/acme/scratch/$CASE/run $ENV{HOME}/projects/acme/scratch/$CASE/bld Windows Subsystem for Linux v2, using Ubuntu distribution none LINUX gnu mpich $ENV{HOME}/e3sm_scratch $ENV{HOME}/pt-e3sm-inputdata $ENV{HOME}/pt-e3sm-inputdata $ENV{HOME}/e3sm_scratch/archive/$CASE $ENV{HOME}/e3sm_baselines $CCSMROOT/tools/cprnc/build/cprnc make 4 e3sm_developer none thorntonpe at ornl dot gov 4 4 mpirun -np {{ total_tasks }} $ENV{HOME}/e3sm_scratch/$CASE/run $ENV{HOME}/e3sm_scratch/$CASE/bld Singularity container singularity LINUX gnu mpich $ENV{HOME}/projects/e3sm/scratch $ENV{HOME}/projects/e3sm/cesm-inputdata $ENV{HOME}/projects/e3sm/ptclm-data $ENV{HOME}/projects/e3sm/scratch/archive/$CASE $ENV{HOME}/projects/e3sm/baselines/$COMPILER $CCSMROOT/tools/cprnc/build/cprnc make 4 e3sm_developer none lukasz at uchicago dot edu 16 16 mpirun -launcher fork -hosts localhost -np {{ total_tasks }} $ENV{HOME}/projects/e3sm/scratch/$CASE/run $ENV{HOME}/projects/e3sm/scratch/$CASE/bld $SRCROOT /usr/local/packages/netcdf-serial /usr/local/packages/cmake/bin:/usr/local/packages/hdf5-serial/bin:/usr/local/packages/netcdf-serial/bin:$ENV{PATH} /usr/local/packages/szip/lib:/usr/local/packages/hdf5-serial/lib:/usr/local/packages/netcdf-serial/lib /usr/local/packages/netcdf-parallel /usr/local/packages/pnetcdf /usr/local/packages/hdf5-parallel /usr/local/packages/cmake/bin:/usr/local/packages/mpich/bin:/usr/local/packages/hdf5-parallel/bin:/usr/local/packages/netcdf-parallel/bin:/usr/local/packages/pnetcdf/bin:$ENV{PATH} /usr/local/packages/mpich/lib:/usr/local/packages/szip/lib:/usr/local/packages/hdf5-parallel/lib:/usr/local/packages/netcdf-parallel/lib:/usr/local/packages/pnetcdf/lib Linux workstation for Jenkins testing (melvin|watson|s999964|climate|penn|sems) LINUX proxy.sandia.gov:80 gnu,intel openmpi /sems-data-store/ACME/timings .* $ENV{HOME}/acme/scratch /sems-data-store/ACME/inputdata /sems-data-store/ACME/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /sems-data-store/ACME/baselines/$COMPILER /sems-data-store/ACME/cprnc/build.new/cprnc 32 e3sm_developer none jgfouca at sandia dot gov 48 48 mpirun -np {{ total_tasks }} --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread:overload-allowed /usr/share/Modules/init/python.py /usr/share/Modules/init/perl.pm /usr/share/Modules/init/sh /usr/share/Modules/init/csh /usr/bin/modulecmd python /usr/bin/modulecmd perl module module sems-env acme-env sems-git acme-binutils sems-python/3.5.2 sems-cmake/3.12.2 sems-gcc/7.3.0 sems-intel/16.0.3 sems-netcdf/4.4.1/exo acme-pfunit/3.2.8/base acme-openmpi/2.1.5 acme-netcdf/4.7.4/acme $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1000 $ENV{SEMS_NETCDF_ROOT} 64M spread threads Huge Linux workstation for Sandia climate scientists mappy LINUX proxy.sandia.gov:80 gnu,gnu9,intel openmpi /sems-data-store/ACME/mappy/timings .* $ENV{HOME}/acme/scratch /sems-data-store/ACME/inputdata /sems-data-store/ACME/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /sems-data-store/ACME/baselines/mappy/$COMPILER /sems-data-store/ACME/mappy/cprnc/cprnc 64 e3sm_developer slurm_single_node jgfouca at sandia dot gov 64 64 srun --cpu_bind=threads /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/env_modules_python.py /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/perl /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/sh /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/init/csh /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/libexec/lmod python /projects/sems/install/rhel7-x86_64/sems/v2/lmod/lmod/8.3/gcc/10.1.0/zbzzu7k/lmod/lmod/libexec/lmod perl module module sems-archive-env acme-env sems-archive-git sems-archive-cmake/3.19.1 acme-gcc/8.1.0 sems-archive-gcc/9.2.0 sems-archive-intel/19.0.5 acme-netcdf/4.4.1/exo_acme acme-pfunit/3.2.8/base sems-archive-netcdf/4.7.3/base acme-openmpi/4.1.4 acme-netcdf/4.7.4/acme $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 0 $ENV{SEMS_NETCDF_ROOT} 64M spread threads Sandia GPU testbed weaver LINUX gnugpu openmpi /home/projects/e3sm/timings .* $ENV{HOME}/acme/scratch /home/projects/e3sm/scream/data /home/projects/e3sm/scream/data/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /home/projects/e3sm/baselines/weaver/$COMPILER /home/projects/e3sm/cprnc/cprnc 32 e3sm_developer lsf jgfouca at sandia dot gov 32 32 mpirun -np {{ total_tasks }} /usr/share/Modules/init/sh /usr/share/Modules/init/python.py module /usr/bin/modulecmd python cuda/10.1.105 ucx/1.6.0 git/2.10.1 python/3.7.3 cmake/3.18.0 perl/5.22.1 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 0 /ascldap/users/projects/e3sm/scream/libs/mpfr/install/weaver/lib:/ascldap/users/projects/e3sm/scream/libs/gcc/install/weaver/gcc/8.5.0/lib64:/ascldap/users/projects/e3sm/scream/libs/gcc/install/weaver/gcc/8.5.0/lib:$ENV{LD_LIBRARY_PATH} /ascldap/users/projects/e3sm/scream/libs/gcc/install/weaver/gcc/8.5.0/bin:/ascldap/users/projects/e3sm/scream/libs/gcc/install/weaver/gcc/8.5.0/libexec/gcc/powerpc64le-unknown-linux-gnu/8.5.0:/ascldap/users/projects/e3sm/scream/libs/openmpi/install/weaver/gcc/8.5.0/cuda/10.1.105/bin:/ascldap/users/projects/e3sm/scream/libs/pnetcdf/install/weaver/gcc/8.5.0/cuda/10.1.105/bin:/ascldap/users/projects/e3sm/scream/libs/netcdf-c/install/weaver/gcc/8.5.0/cuda/10.1.105/bin:/ascldap/users/projects/e3sm/scream/libs/netcdf-fortran/install/weaver/gcc/8.5.0/cuda/10.1.105/bin:/ascldap/users/projects/e3sm/scream/libs/wget/bin:/ascldap/users/jgfouca/perl5/bin:$ENV{PATH} /ascldap/users/jgfouca/perl5/lib/perl5 /ascldap/users/jgfouca/perl5 IBM Power 8 Testbed machine white LINUX gnu openmpi $ENV{HOME}/projects/e3sm/scratch $ENV{HOME}/projects/e3sm/cesm-inputdata $ENV{HOME}/projects/e3sm/ptclm-data $ENV{HOME}/projects/e3sm/scratch/archive/$CASE $ENV{HOME}/projects/e3sm/baselines/$COMPILER $CCSMROOT/tools/cprnc/build/cprnc 32 e3sm_developer lsf mdeakin at sandia dot gov 4 1 mpirun /usr/share/Modules/init/sh /usr/share/Modules/init/python.py module /usr/bin/modulecmd python devpack/20181011/openmpi/2.1.2/gcc/7.2.0/cuda/9.2.88 $ENV{HOME}/projects/e3sm/scratch/$CASE/run $ENV{HOME}/projects/e3sm/scratch/$CASE/bld $ENV{NETCDF_ROOT} /ascldap/users/jgfouca/packages/netcdf-fortran-4.4.4-white $SRCROOT Skylake Testbed machine blake LINUX intel18 openmpi $ENV{HOME}/projects/e3sm/scratch $ENV{HOME}/projects/e3sm/cesm-inputdata $ENV{HOME}/projects/e3sm/ptclm-data $ENV{HOME}/projects/e3sm/scratch/archive/$CASE $ENV{HOME}/projects/e3sm/baselines/$COMPILER $CCSMROOT/tools/cprnc/build/cprnc 48 e3sm_developer slurm mdeakin at sandia dot gov 48 48 mpirun /usr/share/Modules/init/sh /usr/share/Modules/init/python.py module module zlib/1.2.11 intel/compilers/18.1.163 openmpi/2.1.2/intel/18.1.163 hdf5/1.10.1/openmpi/2.1.2/intel/18.1.163 netcdf-exo/4.4.1.1/openmpi/2.1.2/intel/18.1.163 $ENV{HOME}/projects/e3sm/scratch/$CASE/run $ENV{HOME}/projects/e3sm/scratch/$CASE/bld $ENV{NETCDF_ROOT} $ENV{NETCDFF_ROOT} Linux workstation for ANL compute.*mcs.anl.gov LINUX gnu mpich,openmpi $ENV{HOME}/acme/scratch /home/climate1/acme/inputdata /home/climate1/acme/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /home/climate1/acme/baselines/$COMPILER /home/climate1/acme/cprnc/build/cprnc make 32 e3sm_developer none jgfouca at sandia dot gov 32 32 mpirun -l -np {{ total_tasks }} mpirun -np {{ total_tasks }} /software/common/adm/packages/softenv-1.6.2/etc/softenv-load.csh /software/common/adm/packages/softenv-1.6.2/etc/softenv-load.sh source /software/common/adm/packages/softenv-1.6.2/etc/softenv-aliases.csh ; soft source /software/common/adm/packages/softenv-1.6.2/etc/softenv-aliases.sh ; soft +gcc-8.2.0 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /soft/apps/packages/climate/cmake/3.18.4/bin:/soft/apps/packages/climate/gmake/bin:$ENV{PATH} /soft/apps/packages/climate/hdf5/1.8.16-serial/gcc-8.2.0/lib:/soft/apps/packages/climate/szip/2.1/gcc-8.2.0/lib:$ENV{LD_LIBRARY_PATH} /soft/apps/packages/climate/netcdf/4.4.1c-4.2cxx-4.4.4f-serial/gcc-8.2.0 /soft/apps/packages/climate/hdf5/1.8.16-parallel/mpich-3.3.2/gcc-8.2.0/lib:/soft/apps/packages/climate/szip/2.1/gcc-8.2.0/lib:$ENV{LD_LIBRARY_PATH} /soft/apps/packages/climate/mpich/3.3.2/gcc-8.2.0/bin:/soft/apps/packages/climate/cmake/3.18.4/bin:/soft/apps/packages/climate/gmake/bin:$ENV{PATH} /soft/apps/packages/climate/hdf5/1.8.16-parallel/mpich-3.3.2/gcc-8.2.0 /soft/apps/packages/climate/netcdf/4.4.1c-4.2cxx-4.4.4f-parallel/mpich-3.3.2/gcc-8.2.0 /soft/apps/packages/climate/pnetcdf/1.12.0/mpich-3.3.2/gcc-8.2.0 /soft/apps/packages/climate/openmpi/2.1.5/gcc-8.2.0/bin:/soft/apps/packages/climate/cmake/3.18.4/bin:/soft/apps/packages/climate/gmake/bin:$ENV{PATH} /soft/apps/packages/climate/zlib/1.2.11/gcc-8.2.0-static /soft/apps/packages/climate/szip/2.1/gcc-8.2.0-static /soft/apps/packages/climate/hdf5/1.8.12-parallel/openmpi-2.1.5/gcc-8.2.0-static /soft/apps/packages/climate/netcdf/4.7.4c-4.3.1cxx-4.4.4f-parallel/openmpi-2.1.5/gcc-8.2.0-static-hdf5-1.8.12-pnetcdf-1.12.0 /soft/apps/packages/climate/pnetcdf/1.12.0/openmpi-2.1.5/gcc-8.2.0 64M /soft/apps/packages/climate/perl5/lib/perl5 /soft/apps/packages/climate/adios2/2.7.0/mpich-3.3.2/gcc-8.2.0 /soft/apps/packages/climate/adios2/2.7.0/openmpi-2.1.5/gcc-8.2.0 ANL CELS General Computing Environment (Linux) workstation (Ubuntu 18.04) compute-386-01|compute-386-02 LINUX gnu mpich,openmpi /scratch/$ENV{USER}/e3sm/timings .* /scratch/$ENV{USER}/e3sm/scratch /nfs/gce/projects/climate/inputdata $DIN_LOC_ROOT/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /nfs/gce/projects/climate/e3sm/baselines/$COMPILER /nfs/gce/projects/climate/e3sm/cprnc/build/cprnc make 8 e3sm_developer none jayesh at mcs dot anl dot gov 32 32 mpirun -l -np {{ total_tasks }} mpirun --oversubscribe -np {{ total_tasks }} /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.3.0/lmod-7.7.29-zg24dcc/lmod/lmod/init/env_modules_python.py /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.3.0/lmod-7.7.29-zg24dcc/lmod/lmod/init/perl /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.3.0/lmod-7.7.29-zg24dcc/lmod/lmod/init/bash /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.3.0/lmod-7.7.29-zg24dcc/lmod/lmod/init/sh /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.3.0/lmod-7.7.29-zg24dcc/lmod/lmod/init/csh /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.3.0/lmod-7.7.29-zg24dcc/lmod/lmod/libexec/lmod python module module module module autoconf/2.69-tz6eue5 automake/1.16.3-fm5m6qc libtool/2.4.6-jdxbjft m4/1.4.19-wq3bm42 cmake/3.20.5-yjp2hz6 gcc/11.1.0-5ikoznk zlib/1.2.11-smoyzzo $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /nfs/gce/projects/climate/software/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0 /nfs/gce/projects/climate/software/mpich/3.4.2/gcc-11.1.0/lib:$ENV{LD_LIBRARY_PATH} /nfs/gce/projects/climate/software/mpich/3.4.2/gcc-11.1.0/bin:$ENV{PATH} /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/zlib-1.2.11-smoyzzo /nfs/gce/projects/climate/software/hdf5/1.12.1/mpich-3.4.2/gcc-11.1.0 /nfs/gce/projects/climate/software/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/mpich-3.4.2/gcc-11.1.0 /nfs/gce/projects/climate/software/pnetcdf/1.12.2/mpich-3.4.2/gcc-11.1.0 /nfs/gce/projects/climate/software/openmpi/4.1.3/gcc-11.1.0/lib:$ENV{LD_LIBRARY_PATH} /nfs/gce/projects/climate/software/openmpi/4.1.3/gcc-11.1.0/bin:$ENV{PATH} /nfs/gce/software/spack/opt/spack/linux-ubuntu18.04-x86_64/gcc-7.5.0/zlib-1.2.11-smoyzzo /nfs/gce/projects/climate/software/hdf5/1.12.1/openmpi-4.1.3/gcc-11.1.0 /nfs/gce/projects/climate/software/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/openmpi-4.1.3/gcc-11.1.0 /nfs/gce/projects/climate/software/pnetcdf/1.12.2/openmpi-4.1.3/gcc-11.1.0 64M /nfs/gce/projects/climate/software/perl5/lib/perl5 /nfs/gce/projects/climate/software/adios2/2.8.3.patch/mpich-3.4.2/gcc-11.1.0 ANL CELS General Computing Environment (Linux) workstation compute-(240|386)-[0-9][0-9] LINUX gnu mpich,openmpi /scratch/$ENV{USER}/e3sm/timings .* /scratch/$ENV{USER}/e3sm/scratch /nfs/gce/projects/climate/inputdata $DIN_LOC_ROOT/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /nfs/gce/projects/climate/e3sm/baselines/$COMPILER /nfs/gce/projects/climate/e3sm/cprnc/build/cprnc make 8 e3sm_developer none jayesh at mcs dot anl dot gov 32 32 mpirun -l -np {{ total_tasks }} mpirun --oversubscribe -np {{ total_tasks }} /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-6fjdtku/lmod/lmod/init/env_modules_python.py /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-6fjdtku/lmod/lmod/init/perl /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-6fjdtku/lmod/lmod/init/bash /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-6fjdtku/lmod/lmod/init/sh /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-6fjdtku/lmod/lmod/init/csh /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-6fjdtku/lmod/lmod/libexec/lmod python module module module module autoconf/2.69-bmnwajj automake/1.16.3-r7w24o4 libtool/2.4.6-uh3mpsu m4/1.4.19-7fztfyz cmake/3.20.5-zyz2eld gcc/11.1.0-qsjmpcg zlib/1.2.11-p7dmb5p $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-serial/gcc-11.1.0 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/lib:$ENV{LD_LIBRARY_PATH} /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/mpich/4.0/gcc-11.1.0/bin:$ENV{PATH} /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/zlib-1.2.11-p7dmb5p /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/hdf5/1.12.1/mpich-4.0/gcc-11.1.0 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/mpich-4.0/gcc-11.1.0 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/pnetcdf/1.12.2/mpich-4.0/gcc-11.1.0 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/openmpi/4.1.3/gcc-11.1.0/lib:$ENV{LD_LIBRARY_PATH} /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/openmpi/4.1.3/gcc-11.1.0/bin:$ENV{PATH} /nfs/gce/software/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/zlib-1.2.11-p7dmb5p /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/hdf5/1.12.1/openmpi-4.1.3/gcc-11.1.0 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/netcdf/4.8.0c-4.3.1cxx-4.5.3f-parallel/openmpi-4.1.3/gcc-11.1.0 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/pnetcdf/1.12.2/openmpi-4.1.3/gcc-11.1.0 64M /nfs/gce/projects/climate/software/perl5/lib/perl5 /nfs/gce/projects/climate/software/linux-ubuntu20.04-x86_64/adios2/2.8.3.patch/mpich-4.0/gcc-11.1.0 SNL clust (skybridge|chama) LINUX proxy.sandia.gov:80 intel openmpi fy210162 /projects/ccsm/timings .* /gpfs/$USER/acme_scratch/sandiatoss3 /projects/ccsm/inputdata /projects/ccsm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /projects/ccsm/ccsm_baselines/$COMPILER /projects/ccsm/cprnc/build.toss3/cprnc 8 e3sm_integration slurm jgfouca at sandia dot gov 16 16 TRUE mpiexec --n {{ total_tasks }} --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to core /usr/share/lmod/lmod/init/python.py /usr/share/lmod/lmod/init/perl.pm /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod python /usr/share/lmod/lmod/libexec/lmod perl module module sems-archive-env acme-env sems-archive-git sems-archive-cmake/3.19.1 gnu/6.3.1 sems-archive-intel/17.0.0 sems-archive-openmpi/1.10.5 acme-netcdf/4.7.4/acme sems-archive-netcdf/4.4.1/exo /nscratch/$USER/acme_scratch/sandiatoss3/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 $ENV{SEMS_NETCDF_ROOT} $ENV{SEMS_NETCDF_ROOT}/include $ENV{SEMS_NETCDF_ROOT}/lib 64M $ENV{SEMS_NETCDF_ROOT} SNL clust ghost-login LINUX proxy.sandia.gov:80 intel openmpi fy210162 /gscratch/$USER/acme_scratch/ghost /projects/ccsm/inputdata /projects/ccsm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /projects/ccsm/ccsm_baselines/$COMPILER /projects/ccsm/cprnc/build.toss3/cprnc 8 e3sm_integration slurm jgfouca at sandia dot gov 36 36 TRUE mpiexec --n {{ total_tasks }} --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to core /usr/share/lmod/lmod/init/python.py /usr/share/lmod/lmod/init/perl.pm /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod python /usr/share/lmod/lmod/libexec/lmod perl module module sems-env sems-git sems-python/3.5.2 sems-cmake gnu/4.9.2 sems-intel/16.0.2 mkl/16.0 sems-netcdf/4.4.1/exo_parallel sems-openmpi/1.10.5 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld $ENV{SEMS_NETCDF_ROOT} $ENV{SEMS_NETCDF_ROOT}/include $ENV{SEMS_NETCDF_ROOT}/lib 64M $ENV{SEMS_NETCDF_ROOT} ANL/LCRC Linux Cluster (b\d+|blues.*).lcrc.anl.gov LINUX intel,gnu impi,openmpi,mvapich condo /lcrc/group/e3sm .* /lcrc/group/e3sm/$USER/scratch/anvil /lcrc/group/e3sm/public_html/$ENV{USER} https://web.lcrc.anl.gov/public/e3sm/$ENV{USER} /lcrc/group/e3sm/data/inputdata /lcrc/group/e3sm/data/inputdata/atm/datm7 /lcrc/group/e3sm/$USER/archive/$CASE /lcrc/group/e3sm/baselines/anvil/$COMPILER /lcrc/group/e3sm/soft/tools/cprnc/cprnc 8 e3sm_integration slurm E3SM 36 36 FALSE srun -l -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit --cpu_bind=cores -c $ENV{OMP_NUM_THREADS} -m plane={{ tasks_per_node }} /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/sh;export MODULEPATH=$MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/csh;setenv MODULEPATH $MODULEPATH\:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/env_modules_python.py export MODULEPATH=$MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core;/home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/libexec/lmod python module module cmake/3.20.3-vedypwm gcc/7.4.0 intel/20.0.4-lednsve intel-mkl/2020.4.304-voqlapk mvapich2/2.3.6-verbs-x4iz7lq netcdf-c/4.4.1-gei7x7w netcdf-cxx/4.2-db2f5or netcdf-fortran/4.4.4-b4ldb3a parallel-netcdf/1.11.0-kj4jsvt intel-mpi/2019.9.304-i42whlw netcdf-c/4.4.1-blyisdg netcdf-cxx/4.2-gkqc6fq netcdf-fortran/4.4.4-eanrh5t parallel-netcdf/1.11.0-y3nmmej openmpi/4.1.1-v3b3npd netcdf-c/4.4.1-smyuxme netcdf-cxx/4.2-kfb2aag netcdf-fortran/4.4.4-mablvyc parallel-netcdf/1.11.0-x4n5s7k gcc/8.2.0-xhxgy33 intel-mkl/2020.4.304-d6zw4xa netcdf/4.4.1-ve2zfkw netcdf-cxx/4.2-2rkopdl netcdf-fortran/4.4.4-thtylny mvapich2/2.2-verbs-ppznoge parallel-netcdf/1.11.0-c22b2bn intel-mpi/2019.9.304-rxpzd6p netcdf-c/4.4.1-fysjgfx netcdf-cxx/4.2-oaiw2v6 netcdf-fortran/4.4.4-kxgkaop parallel-netcdf/1.11.0-fce7akl openmpi/4.1.1-x5n4m36 netcdf-c/4.4.1-mtfptpl netcdf-cxx/4.2-osp27dq netcdf-fortran/4.4.4-5yd6dos parallel-netcdf/1.11.0-a7ohxsg $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1000 $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf_version))} /lcrc/group/e3sm/soft/perl/5.26.0/bin:$ENV{PATH} 0 1 1 1 2 10 64M granularity=core,balanced 1 cores /lcrc/group/e3sm/3rdparty/anvil/adios2/2.8.3.patch/intel-mpi-2019.9.304/intel-20.0.4 /lcrc/group/e3sm/3rdparty/anvil/adios2/2.8.3.patch/intel-mpi-2019.9.304/gcc-8.2.0 /lcrc/group/e3sm/3rdparty/anvil/adios2/2.8.3.patch/openmpi-4.1.1/intel-20.0.4 /lcrc/group/e3sm/3rdparty/anvil/adios2/2.8.3.patch/openmpi-4.1.1/gcc-8.2.0 /lcrc/group/e3sm/3rdparty/anvil/adios2/2.8.3.patch/mvapich2-2.3.6/intel-20.0.4 /lcrc/group/e3sm/3rdparty/anvil/adios2/2.8.3.patch/mvapich2-2.2/gcc-8.2.0 ANL LCRC cluster 512-node AMD Epyc 7532 2-sockets 64-cores per node chr.* LINUX intel,gnu,oneapi-ifx openmpi,impi e3sm /lcrc/group/e3sm/PERF_Chrysalis .* /lcrc/group/e3sm/$USER/scratch/chrys /lcrc/group/e3sm/public_html/$ENV{USER} https://web.lcrc.anl.gov/public/e3sm/$ENV{USER} /lcrc/group/e3sm/data/inputdata /lcrc/group/e3sm/data/inputdata/atm/datm7 /lcrc/group/e3sm/$USER/scratch/chrys/archive/$CASE /lcrc/group/e3sm/baselines/chrys/$COMPILER /lcrc/group/e3sm/tools/cprnc/cprnc 8 e3sm_integration 4 slurm E3SM 128 64 FALSE srun --mpi=pmi2 -l -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit $SHELL{if [ 64 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -c $SHELL{echo 128/ {{ tasks_per_node }} |bc} -m plane={{ tasks_per_node }} /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/lmod-8.3-5be73rg/lmod/lmod/init/sh /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/lmod-8.3-5be73rg/lmod/lmod/init/csh /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/lmod-8.3-5be73rg/lmod/lmod/init/env_modules_python.py /gpfs/fs1/soft/chrysalis/spack/opt/spack/linux-centos8-x86_64/gcc-9.3.0/lmod-8.3-5be73rg/lmod/lmod/libexec/lmod python module module subversion/1.14.0-e4smcy3 perl/5.32.0-bsnc6lt cmake/3.24.2-whgdv7y intel/20.0.4-kodw73g intel-mkl/2020.4.304-g2qaxzf openmpi/4.1.3-pin4k7o hdf5/1.10.7-eewgp6v netcdf-c/4.4.1-ihoo4zq netcdf-cxx/4.2-soitsxm netcdf-fortran/4.4.4-tplolxh parallel-netcdf/1.11.0-gvcfihh intel-mpi/2019.9.304-tkzvizk hdf5/1.8.16-se4xyo7 netcdf-c/4.4.1-qvxyzq2 netcdf-cxx/4.2-binixgj netcdf-fortran/4.4.4-rdxohvp parallel-netcdf/1.11.0-b74wv4m gcc/9.2.0-ugetvbp intel-mkl/2020.4.304-n3b5fye openmpi/4.1.3-sxfyy4k hdf5/1.10.7-j3zxncu netcdf-c/4.4.1-7ohuiwq netcdf-cxx/4.2-tkg465k netcdf-fortran/4.4.4-k2zu3y5 parallel-netcdf/1.11.0-mirrcz7 intel-mpi/2019.9.304-jdih7h5 hdf5/1.8.16-dtbpce3 netcdf-c/4.4.1-zcoa44z netcdf-cxx/4.2-ayxg4c7 netcdf-fortran/4.4.4-2lfr2lr parallel-netcdf/1.11.0-ifdodru intel-oneapi-compilers/2022.2.0-dioefq5 intel-oneapi-mkl/2022.2.0-w2y75l6 openmpi/4.1.3-ti25nay hdf5/1.10.7-clocd2t netcdf-c/4.7.4-nw7dztf netcdf-fortran/4.5.3-qa6rhpj parallel-netcdf/1.11.0-ejxy7kk $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.05 0.05 1000 /lcrc/group/e3sm/soft/perl/chrys/lib/perl5 $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf_version))} 128M granularity=core,balanced granularity=thread,balanced cores /lcrc/group/e3sm/3rdparty/chrysalis/adios2/2.8.3.patch/openmpi-4.1.3/intel-20.0.4 /lcrc/group/e3sm/3rdparty/chrysalis/adios2/2.8.3.patch/openmpi-4.1.3/gcc-9.2.0 /lcrc/group/e3sm/3rdparty/chrysalis/adios2/2.8.3.patch/intel-mpi-2019.9.304/intel-20.0.4 /lcrc/group/e3sm/3rdparty/chrysalis/adios2/2.8.3.patch/intel-mpi-2019.9.304/gcc-9.2.0 ANL/LCRC Linux Cluster LINUX pgigpu mvapich e3sm /lcrc/group/e3sm .* /lcrc/group/e3sm/$USER/scratch/blues /lcrc/group/e3sm/data/inputdata /lcrc/group/e3sm/data/inputdata/atm/datm7 /lcrc/group/e3sm/$USER/archive/$CASE /lcrc/group/e3sm/baselines/blues/$COMPILER /lcrc/group/e3sm/soft/tools/cprnc/cprnc 8 e3sm_integration 4 slurm E3SM 16 16 TRUE srun -l -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit --cpu_bind=cores -c $ENV{OMP_NUM_THREADS} -m plane=$SHELL{echo 16/$OMP_NUM_THREADS|bc} /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/sh;export MODULEPATH=$MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core:/blues/gpfs/home/software/spack-0.10.1/share/spack/lmod/linux-centos7-x86_64/Core /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/csh;setenv MODULEPATH $MODULEPATH\:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core\:/blues/gpfs/home/software/spack-0.10.1/share/spack/lmod/linux-centos7-x86_64/Core /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/env_modules_python.py export MODULEPATH=$MODULEPATH:/software/centos7/spack-latest/share/spack/lmod/linux-centos7-x86_64/Core:/blues/gpfs/home/software/spack-0.10.1/share/spack/lmod/linux-centos7-x86_64/Core;/home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/libexec/lmod python module module cmake/3.20.3-vedypwm nvhpc/20.9-5brtudu cuda/11.1.0-6dvax5z netcdf-c/4.7.4-ltqliri netcdf-cxx/4.2-kf5ox4e netcdf-fortran/4.5.3-6mgyroo mvapich2/2.3.4-blues-5fwicb5 parallel-netcdf/1.12.1-nyuvwhn $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1000 $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf_version))} /lcrc/group/e3sm/soft/perl/5.26.0/bin:$ENV{PATH} 0 1 1 2 64M cores ANL/LCRC Linux Cluster: 6x 128c EPYC nodes with 8x A100 GPUs gpulogin.* LINUX pgigpu openmpi e3sm /lcrc/group/e3sm .* /lcrc/group/e3sm/$USER/scratch/swing /lcrc/group/e3sm/data/inputdata /lcrc/group/e3sm/data/inputdata/atm/datm7 /lcrc/group/e3sm/$USER/archive/$CASE /lcrc/group/e3sm/baselines/swing/$COMPILER /lcrc/group/e3sm/soft/tools/cprnc/cprnc 8 e3sm_gpu 4 slurm E3SM 128 16 128 16 TRUE srun -l -n {{ total_tasks }} -N {{ num_nodes }} -K $SHELL{if [ 128 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -c $SHELL{echo 256/ {{ tasks_per_node }} |bc} -m plane={{ tasks_per_node }} /gpfs/fs1/soft/swing/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-5tuyfdb/lmod/lmod/init/sh /gpfs/fs1/soft/swing/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-5tuyfdb/lmod/lmod/init/csh /gpfs/fs1/soft/swing/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-5tuyfdb/lmod/lmod/init/env_modules_python.py /gpfs/fs1/soft/swing/spack/opt/spack/linux-ubuntu20.04-x86_64/gcc-9.3.0/lmod-8.3-5tuyfdb/lmod/lmod/libexec/lmod python module module cmake/3.21.1-e5i6eks nvhpc/20.9-37zsymt cuda/11.1.1-nkh7mm7 openmpi/4.1.1-r6ebr2e netcdf-c/4.7.4-zppo53l netcdf-cxx/4.2-wjm7fye netcdf-fortran/4.5.3-srsajjs parallel-netcdf/1.12.1-75szceu $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1000 $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf_version))} /lcrc/group/e3sm/soft/perl/5.26.0/bin:$ENV{PATH} 64M cores ANL/LCRC Cluster, Cray CS400, 352-nodes Xeon Phi 7230 KNLs 64C/1.3GHz + 672-nodes Xeon E5-2695v4 Broadwells 36C/2.10GHz, Intel Omni-Path network, SLURM batch system, Lmod module environment. beboplogin.* LINUX intel,gnu impi,mvapich e3sm /lcrc/group/e3sm/$USER/scratch/bebop /lcrc/group/e3sm/data/inputdata /lcrc/group/e3sm/data/inputdata/atm/datm7 /lcrc/group/e3sm/$USER/archive/$CASE /lcrc/group/e3sm/baselines/bebop/$COMPILER /lcrc/group/e3sm/soft/tools/cprnc/cprnc 8 e3sm_integration 4 slurm E3SM 36 36 TRUE mpirun -l -n {{ total_tasks }} srun -l -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit --cpu_bind=cores -c $ENV{OMP_NUM_THREADS} -m plane=$SHELL{echo 36/$OMP_NUM_THREADS|bc} /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/sh /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/csh /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/init/env_modules_python.py /home/software/spack-0.10.1/opt/spack/linux-centos7-x86_64/gcc-4.8.5/lmod-7.4.9-ic63herzfgw5u3na5mdtvp3nwxy6oj2z/lmod/lmod/libexec/lmod python module module cmake/3.20.3-vedypwm anaconda3/5.2.0 intel/18.0.4-443hhug intel-mkl/2018.4.274-jwaeshj hdf5/1.10.5-3mk3uik netcdf/4.7.0-krelxcz netcdf-fortran/4.4.5-74lj75q intel-mpi/2018.4.274-4hmwfl6 parallel-netcdf/1.11.0-acswzws mvapich2/2.3.1-verbs-omjz3ck parallel-netcdf/1.11.2-7fy6qz3 gcc/8.2.0-g7hppkz intel-mkl/2018.4.274-2amycpi hdf5/1.8.16-mz7lmxh netcdf/4.4.1-xkjcghm netcdf-fortran/4.4.4-mpstomu intel-mpi/2018.4.274-ozfo327 parallel-netcdf/1.11.0-filvnis mvapich2/2.3-bebop-3xi4hiu parallel-netcdf/1.11.2-hfn33fd $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} /lcrc/group/e3sm/soft/perl/5.26.0/bin:$ENV{PATH} $SHELL{dirname $(dirname $(which pnetcdf_version))} $SHELL{which h5dump | xargs dirname | xargs dirname} 128M spread threads shm:tmi /lcrc/group/e3sm/3rdparty/bebop/adios2/2.8.3.patch/intel-mpi-2018.4.274/intel-18.0.4 /lcrc/group/e3sm/3rdparty/bebop/adios2/2.8.3.patch/intel-mpi-2018.4.274/gcc-8.2.0 /lcrc/group/e3sm/3rdparty/bebop/adios2/2.8.3.patch/mvapich2-2.3.1/intel-18.0.4 /lcrc/group/e3sm/3rdparty/bebop/adios2/2.8.3.patch/mvapich2-2.3/gcc-8.2.0 LLNL Linux Cluster, Linux (pgi), 16 pes/node, batch system is Slurm LINUX intel mpich cbronze /p/lustre2/$USER/e3sm_scratch/syrah /usr/gdata/climdat/ccsm3data/inputdata /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER /usr/gdata/climdat/tools/cprnc 8 lc_slurm donahue5 -at- llnl.gov 32 16 srun /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh module module /usr/share/lmod/lmod/libexec/lmod python /usr/share/lmod/lmod/libexec/lmod perl python git intel/19.0.4 mvapich2/2.3 cmake/3.18.0 netcdf-fortran/4.4.4 pnetcdf/1.9.0 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /usr/tce/packages/netcdf-fortran/netcdf-fortran-4.4.4-intel-19.0.4/ /usr/tce/packages/netcdf-fortran/netcdf-fortran-4.4.4-intel-19.0.4/ /usr/tce/packages/pnetcdf/pnetcdf-1.9.0-intel-19.0.4-mvapich2-2.3/ LLNL Linux Cluster, Linux (pgi), 56 pes/node, batch system is Slurm LINUX intel mpich cbronze /p/lustre2/$USER/e3sm_scratch/ruby /usr/gdata/climdat/ccsm3data/inputdata /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER /usr/gdata/climdat/tools/cprnc 8 lc_slurm donahue5 -at- llnl.gov 56 56 srun /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh module module /usr/share/lmod/lmod/libexec/lmod python /usr/share/lmod/lmod/libexec/lmod perl python/3.8.2 git intel/19.0.4 mvapich2/2.3 cmake/3.18.0 netcdf-fortran/4.4.4 pnetcdf/1.9.0 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /usr/tce/packages/netcdf-fortran/netcdf-fortran-4.4.4-intel-19.0.4/ /usr/tce/packages/netcdf-fortran/netcdf-fortran-4.4.4-intel-19.0.4/ /usr/tce/packages/pnetcdf/pnetcdf-1.9.0-intel-19.0.4-mvapich2-2.3/ LLNL Linux Cluster, Linux (pgi), 36 pes/node, batch system is Slurm LINUX intel mpich cbronze /p/lustre2/$USER/e3sm_scratch/quartz /usr/gdata/climdat/ccsm3data/inputdata /usr/gdata/climdat/ccsm3data/inputdata/atm/datm7 /p/lustre2/$USER/archive/$CASE /p/lustre2/$USER/ccsm_baselines/$COMPILER /usr/gdata/climdat/tools/cprnc 8 lc_slurm donahue5 -at- llnl.gov 72 36 srun /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/perl /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh module module /usr/share/lmod/lmod/libexec/lmod python /usr/share/lmod/lmod/libexec/lmod perl python/3.9.12 git mkl/2022.1.0 intel-classic/2021.6.0-magic mvapich2/2.3.6 cmake/3.19.2 netcdf-fortran-parallel/4.6.0 netcdf-c-parallel/4.9.0 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /usr/tce/packages/netcdf-fortran/netcdf-fortran-4.6.0-mvapich2-2.3.6-intel-classic-2021.6.0/ /usr/tce/packages/netcdf-fortran/netcdf-fortran-4.6.0-mvapich2-2.3.6-intel-classic-2021.6.0/ ALCF Cray XC40 KNL, os is CNL, 64 pes/node, batch system is cobalt theta.* CNL intel,gnu,cray mpt /projects/$PROJECT ClimateEnergy_4 /projects/$PROJECT/$USER /projects/ccsm/e3sm/inputdata /projects/ccsm/e3sm/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /projects/$PROJECT/e3sm/baselines/$COMPILER /projects/ccsm/e3sm/tools/cprnc/cprnc 16 e3sm_developer 4 cobalt_theta E3SM 128 64 TRUE aprun -n {{ total_tasks }} -N $SHELL{if [ `./xmlquery --value MAX_MPITASKS_PER_NODE` -gt `./xmlquery --value TOTAL_TASKS` ];then echo `./xmlquery --value TOTAL_TASKS`;else echo `./xmlquery --value MAX_MPITASKS_PER_NODE`;fi;} --cc depth -d $SHELL{echo `./xmlquery --value MAX_TASKS_PER_NODE`/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} -j $SHELL{if [ 64 -ge `./xmlquery --value MAX_TASKS_PER_NODE` ];then echo 1;else echo `./xmlquery --value MAX_TASKS_PER_NODE`/64|bc;fi;} $ENV{SMP_VARS} $ENV{labeling} /opt/modules/default/init/perl.pm /opt/modules/default/init/python.py /opt/modules/default/init/sh /opt/modules/default/init/csh /opt/modules/default/bin/modulecmd perl /opt/modules/default/bin/modulecmd python module module cray-mpich cray-parallel-netcdf cray-hdf5-parallel cray-hdf5 cray-netcdf cray-netcdf-hdf5parallel craype/2.6.5 cmake/3.18.0 PrgEnv-gnu PrgEnv-cray PrgEnv-intel/6.0.7 intel/19.1.0.166 PrgEnv-intel PrgEnv-cray PrgEnv-gnu/6.0.7 gcc/9.3.0 PrgEnv-intel PrgEnv-gnu gcc/9.3.0 PrgEnv-cray/6.0.9 cce/10.0.3 darshan cray-libsci/20.09.1 cray-mpich/7.7.14 cray-hdf5-parallel/1.10.6.1 cray-netcdf-hdf5parallel/4.7.3.3 cray-parallel-netcdf/1.12.0.1 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1000 /projects/ccsm/e3sm/soft/perl/5.26.0/bin:$ENV{PATH} /projects/ccsm/e3sm/tools/mpas 1 -e PMI_LABEL_ERROUT=1 -e OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} -e OMP_STACKSIZE=128M -e KMP_AFFINITY=granularity=thread,scatter -e OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} -e OMP_STACKSIZE=128M -e OMP_PROC_BIND=spread -e OMP_PLACES=threads /projects/ClimateEnergy_4/3rdparty/adios2/2.7.0/cray-mpich-7.7.14/intel-19.1.0 /projects/ClimateEnergy_4/3rdparty/adios2/2.7.0/cray-mpich-7.7.14/gcc-9.3.0 ANL experimental/evaluation cluster, batch system is cobalt jlse.* LINUX oneapi-ifx,oneapi-ifxgpu,oneapi-ifort,gnu mpich,impi,openmpi /gpfs/jlse-fs0/projects/climate/$USER/scratch /gpfs/jlse-fs0/projects/climate/inputdata /gpfs/jlse-fs0/projects/climate/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /gpfs/jlse-fs0/projects/climate/baselines/$COMPILER /gpfs/jlse-fs0/projects/climate/tools/cprnc/cprnc 16 e3sm_developer 4 cobalt_theta e3sm 112 96 96 112 48 48 FALSE mpirun -l -n {{ total_tasks }} -bind-to core mpirun --tag-output -n {{ total_tasks }} --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread /usr/share/Modules/init/sh /usr/share/Modules/init/csh /usr/share/Modules/init/python.py module module /usr/bin/modulecmd python /soft/modulefiles cmake/3.22.1 /soft/restricted/CNDA/modules oneapi/eng-compiler/2022.10.15.006 cmake gcc/8.2.0 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /home/azamat/soft/perl/5.32.0/bin:$ENV{PATH} /home/azamat/soft/netcdf/4.4.1c-4.2cxx-4.4.4f/oneapi-2020.12.15.004-intel_mpi-2019.4.243 /home/azamat/soft/pnetcdf/1.12.1/oneapi-2020.12.15.004-intel_mpi-2019.4.243 1 10 omp spread unit icc icpc ifort /home/azamat/soft/openmpi/2.1.6/intel19/bin:$ENV{PATH} /home/azamat/soft/openmpi/2.1.6/intel19/lib:$ENV{LD_LIBRARY_PATH} /home/azamat/soft/netcdf/4.4.1c-4.2cxx-4.4.4f/intel19-openmpi2.1.6 /home/azamat/soft/pnetcdf/1.12.1/intel19-openmpi2.1.6 gcc g++ gfortran /home/azamat/soft/openmpi/2.1.6/gcc8.2.0/lib:/home/azamat/soft/libs:$ENV{LD_LIBRARY_PATH} /home/azamat/soft/openmpi/2.1.6/gcc8.2.0/bin:/home/azamat/soft/cmake/3.18.5/bin:$ENV{PATH} /home/azamat/soft/cmake/3.18.5 /home/azamat/soft/cmake/3.18.5/share/aclocal /home/azamat/soft/cmake/3.18.5 /home/azamat/soft/netcdf/4.4.1c-4.2cxx-4.4.4f/gcc8.2.0-openmpi2.1.6 /home/azamat/soft/pnetcdf/1.12.1/gcc8.2.0-openmpi2.1.6 opencl NO_GPU 0 4000MB 0 0 DISABLED verbose,granularity=thread,balanced 128M threads 128M -1 ANL Sunspot Test and Development System (TDS), batch system is pbspro uan-.* LINUX oneapi-ifx,oneapi-ifxgpu,oneapi-ifort,gnu mpich,impi,openmpi CSC249ADSE15_CNDA /gila/CSC249ADSE15_CNDA/performance_archive .* /lus/gila/projects/CSC249ADSE15_CNDA/$USER/scratch /lus/gila/projects/CSC249ADSE15_CNDA/inputdata /lus/gila/projects/CSC249ADSE15_CNDA/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /lus/gila/projects/CSC249ADSE15_CNDA/baselines/$COMPILER /lus/gila/projects/CSC249ADSE15_CNDA/tools/cprnc/cprnc 16 e3sm_developer 4 pbspro e3sm 208 208 104 104 104 12 FALSE mpiexec -np {{ total_tasks }} --label -ppn {{ tasks_per_node }} --cpu-bind depth -envall -d $ENV{OMP_NUM_THREADS} $ENV{GPU_TILE_COMPACT} mpirun --tag-output -n {{ total_tasks }} --map-by ppr:{{ tasks_per_numa }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread /soft/packaging/lmod/lmod/init/sh /soft/packaging/lmod/lmod/init/csh /soft/packaging/lmod/lmod/init/env_modules_python.py module module /soft/packaging/lmod/lmod/libexec/lmod python /soft/modulefiles spack cmake /soft/restricted/CNDA/updates/modulefiles oneapi/eng-compiler/2022.12.30.003 mpich/52.2/icc-all-pmix-gpu spack cmake gcc/10.3.0 cray-pals append-deps/default libfabric/1.15.2.0 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/netcdf /lus/gila/projects/CSC249ADSE15_CNDA/software/oneAPI.2022.12.30.003/pnetcdf 1 10 omp spread unit level_zero:gpu NO_GPU 0 disable disable 1 4000MB 0 /soft/tools/mpi_wrapper_utils/gpu_tile_compact.sh 131072 20 0 DISABLED 131072 20 0 verbose,granularity=thread,balanced 128M threads 128M -1 PNL cluster, OS is Linux, batch system is SLURM sooty LINUX intel,pgi mvapich2 /lustre/$USER/cime_output_root /pic/projects/sooty2/$ENV{USER}/e3sm_inputdata /pic/projects/sooty2/$ENV{USER}/e3sm_inputdata/atm/datm7 /lustre/$USER/archive/$CASE /lustre/climate/acme_baselines/$COMPILER /lustre/climate/acme_baselines/cprnc/cprnc 8 slurm balwinder.singh -at- pnnl.gov 8 8 FALSE srun --mpi=none --ntasks={{ total_tasks }} --cpu_bind=sockets --cpu_bind=verbose --kill-on-bad-exit /share/apps/modules/Modules/3.2.10/init/perl.pm /share/apps/modules/Modules/3.2.10/init/python.py /etc/profile.d/modules.csh /etc/profile.d/modules.sh /share/apps/modules/Modules/3.2.10/bin/modulecmd perl /share/apps/modules/Modules/3.2.10/bin/modulecmd python module module perl/5.20.0 cmake/3.17.1 svn/1.8.13 intel/19.0.5 mkl/2019u5 pgi/14.10 mvapich2/2.3.1 netcdf/4.6.3 /lustre/$USER/csmruns/$CASE/run /lustre/$USER/csmruns/$CASE/bld $ENV{MKLROOT} $ENV{NETCDF_LIB}/../ 64M /share/apps/python/3.7.2/lib/:/share/apps/openssl/1.0.2r/lib:$ENV{LD_LIBRARY_PATH}:/share/apps/gcc/8.1.0/lib:/share/apps/gcc/8.1.0/lib64: /share/apps/python/3.7.2/bin:$ENV{PATH} PNNL Intel KNC cluster, OS is Linux, batch system is SLURM glogin LINUX intel impi,mvapich2 /dtemp/$PROJECT/$USER /dtemp/st49401/sing201/acme/inputdata/ /dtemp/st49401/sing201/acme/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE $CIME_OUTPUT_ROOT/acme/acme_baselines $CIME_OUTPUT_ROOT/acme/acme_baselines/cprnc/cprnc 8 slurm balwinder.singh -at- pnnl.gov 16 16 TRUE mpirun -np {{ total_tasks }} srun --mpi=none --ntasks={{ total_tasks }} --cpu_bind=sockets --cpu_bind=verbose --kill-on-bad-exit /opt/lmod/7.8.4/init/env_modules_python.py /etc/profile.d/modules.csh /etc/profile.d/modules.sh /opt/lmod/7.8.4/libexec/lmod python module module python/2.7.9 intel/ips_18 mkl/14.0 impi/4.1.2.040 mvapich2/1.9 netcdf/4.3.0 $CIME_OUTPUT_ROOT/csmruns/$CASE/run $CIME_OUTPUT_ROOT/csmruns/$CASE/bld 64M $ENV{NETCDF_ROOT} $ENV{MLIBHOME} intel PNL Haswell cluster, OS is Linux, batch system is SLURM constance LINUX intel,pgi,nag mvapich2,openmpi,intelmpi,mvapich /pic/scratch/$USER /pic/projects/climate/csmdata/ /pic/projects/climate/csmdata/atm/datm7 /pic/scratch/$USER/archive/$CASE /pic/projects/climate/acme_baselines/$COMPILER /pic/projects/climate/acme_baselines/cprnc/cprnc 8 slurm balwinder.singh -at- pnnl.gov 24 24 FALSE srun --mpi=none --ntasks={{ total_tasks }} --cpu_bind=sockets --cpu_bind=verbose --kill-on-bad-exit srun --ntasks={{ total_tasks }} --cpu_bind=sockets --cpu_bind=verbose --kill-on-bad-exit mpirun -n {{ total_tasks }} mpirun -n {{ total_tasks }} /share/apps/modules/Modules/3.2.10/init/perl.pm /share/apps/modules/Modules/3.2.10/init/python.py /etc/profile.d/modules.csh /etc/profile.d/modules.sh /share/apps/modules/Modules/3.2.10/bin/modulecmd perl /share/apps/modules/Modules/3.2.10/bin/modulecmd python module module perl/5.20.0 cmake/3.17.1 gcc/8.1.0 intel/19.0.5 mkl/2019u5 pgi/14.10 nag/6.0 mkl/15.0.1 mvapich2/2.1 mvapich2/2.3.1 mvapich2/2.1 mvapich2/2.3b intelmpi/5.0.1.035 openmpi/1.8.3 netcdf/4.6.3 netcdf/4.3.2 netcdf/4.4.1.1 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /share/apps/gcc/8.1.0/lib:/share/apps/gcc/8.1.0/lib64:$ENV{LD_LIBRARY_PATH} 64M $ENV{NETCDF_LIB}/../ $ENV{MKLROOT} $ENV{MKLROOT} PNL E3SM Intel Xeon Gold 6148(Skylake) nodes, OS is Linux, SLURM compy LINUX intel,pgi impi,mvapich2 /compyfs .* /compyfs/$USER/e3sm_scratch /compyfs/inputdata /compyfs/inputdata/atm/datm7 /compyfs/$USER/e3sm_scratch/archive/$CASE /compyfs/e3sm_baselines/$COMPILER /compyfs/e3sm_baselines/cprnc/cprnc 8 e3sm_integration 4 slurm bibi.mathew -at- pnnl.gov 40 40 TRUE srun --mpi=none --ntasks={{ total_tasks }} --nodes={{ num_nodes }} --kill-on-bad-exit -l --cpu_bind=cores -c $ENV{OMP_NUM_THREADS} -m plane=$SHELL{echo 40/$OMP_NUM_THREADS|bc} srun --mpi=pmi2 --ntasks={{ total_tasks }} --nodes={{ num_nodes }} --kill-on-bad-exit -l --cpu_bind=cores -c $ENV{OMP_NUM_THREADS} -m plane=$SHELL{echo 40/$OMP_NUM_THREADS|bc} /share/apps/modules/init/perl.pm /share/apps/modules/init/python.py /etc/profile.d/modules.csh /etc/profile.d/modules.sh /share/apps/modules/bin/modulecmd perl /share/apps/modules/bin/modulecmd python module module cmake/3.19.6 gcc/8.1.0 intel/19.0.5 pgi/19.10 mvapich2/2.3.1 intelmpi/2019u4 intelmpi/2019u3 netcdf/4.6.3 pnetcdf/1.9.0 mkl/2019u5 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.05 0 $ENV{NETCDF_ROOT}/ $ENV{MKLROOT} /share/apps/gcc/8.1.0/lib:/share/apps/gcc/8.1.0/lib64:$ENV{LD_LIBRARY_PATH} 0 1 1 10 64M cores /qfs/people/wuda020/3rdparty/adios2/2.8.3.patch/intelmpi-2019u4/intel-19.0.5 /qfs/people/wuda020/3rdparty/adios2/2.8.3.patch/intelmpi-2019u3/pgi-19.10 /qfs/people/wuda020/3rdparty/adios2/2.8.3.patch/mvapich2-2.3.1/intel-19.0.4 /qfs/people/wuda020/3rdparty/adios2/2.8.3.patch/mvapich2-2.3.1/pgi-19.7 EMSL, OS is Linux, SLURM tlogin LINUX intel impi,mvapich2 /tahoma/emsls60153 .* /tahoma/emsls60153/$USER/e3sm_scratch /tahoma/emsls60153/inputdata /tahoma/emsls60153/inputdata/atm/datm7 /tahoma/emsls60153/$USER/e3sm_scratch/archive/$CASE /tahoma/emsls60153/e3sm_baselines/$COMPILER /tahoma/emsls60153/e3sm_baselines/cprnc/cprnc 8 e3sm_integration 4 slurm balwinder.singh -at- pnnl.gov 36 36 TRUE srun --mpi=none --ntasks={{ total_tasks }} --nodes={{ num_nodes }} --kill-on-bad-exit -l --cpu_bind=cores -c $ENV{OMP_NUM_THREADS} -m plane=$SHELL{echo 40/$OMP_NUM_THREADS|bc} srun --mpi=pmi2 --ntasks={{ total_tasks }} --nodes={{ num_nodes }} --kill-on-bad-exit -l --cpu_bind=cores -c $ENV{OMP_NUM_THREADS} -m plane=$SHELL{echo 40/$OMP_NUM_THREADS|bc} /opt/lmod/7.8.4/init/env_modules_python.py /etc/profile.d/modules.csh /etc/profile.d/modules.sh /opt/lmod/7.8.4/libexec/lmod python module module cmake/3.19.5-intel gcc/10.2.0 intel/ips_20_u2 mvapich2~cuda/2.3.5-intel impi/ips_20_u2 netcdf-c/4.7.4-intel-intel-mpi-2019.10.317 netcdf-fortran/4.5.3-intel-intel-mpi-2019.10.317 mkl/scalapack $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.05 0 $SHELL{dirname $(dirname $(which h5diff))} $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $ENV{MKLROOT} 0 1 1 $SHELL{which libpmi2.so} 10 64M cores ORNL XK6, os is Linux, 32 pes/node, batch system is PBS oic5 LINUX gnu mpich,openmpi /home/$USER/models/ACME /home/zdr/models/ccsm_inputdata /home/zdr/models/ccsm_inputdata/atm/datm7 /home/$USER/models/ACME/run/archive/$CASE 32 e3sm_developer pbs dmricciuto 32 32 /projects/cesm/devtools/mpich-3.0.4-gcc4.8.1/bin/mpirun -np {{ total_tasks }} --hostfile $ENV{PBS_NODEFILE} /home/$USER/models/ACME/run/$CASE/run /home/$USER/models/ACME/run/$CASE/bld OR-CONDO, CADES-CCSI, os is Linux, 16 pes/nodes, batch system is PBS or-condo LINUX gnu,intel openmpi /lustre/or-hydra/cades-ccsi/scratch/$USER /lustre/or-hydra/cades-ccsi/proj-shared/project_acme/ACME_inputdata /lustre/or-hydra/cades-ccsi/proj-shared/project_acme/ACME_inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /lustre/or-hydra/cades-ccsi/proj-shared/project_acme/baselines/$COMPILER /lustre/or-hydra/cades-ccsi/proj-shared/tools/cprnc.orcondo 4 e3sm_developer slurm yinj -at- ornl.gov 32 32 FALSE mpirun -np {{ total_tasks }} /usr/share/Modules/init/sh /usr/share/Modules/init/csh /usr/share/Modules/init/perl.pm /usr/share/Modules/init/python.py module module /usr/bin/modulecmd perl /usr/bin/modulecmd python PE-gnu mkl/2017 cmake/3.12.0 python/2.7.12 nco/4.6.9 hdf5-parallel/1.8.17 netcdf-hdf5parallel/4.3.3.1 pnetcdf/1.9.0 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /software/user_tools/current/cades-ccsi/petsc4pf/openmpi-1.10-gcc-5.3 /software/user_tools/current/cades-ccsi/perl5/lib/perl5/ LANL Linux Cluster, 36 pes/node, batch system slurm gr-fe.*.lanl.gov LINUX intel,gnu openmpi,impi,mvapich climateacme /lustre/scratch4/turquoise/$ENV{USER}/E3SM/scratch /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/atm/datm7 /lustre/scratch4/turquoise/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER /turquoise/usr/projects/climate/SHARED_CLIMATE/software/wolf/cprnc/v0.40/cprnc 4 e3sm_developer slurm luke.vanroekel @ gmail.com 36 32 TRUE srun -n {{ total_tasks }} /usr/share/Modules/init/perl.pm /usr/share/Modules/init/python.py /etc/profile.d/z00_lmod.sh /etc/profile.d/z00_lmod.csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module module cmake/3.16.2 gcc/6.4.0 openmpi/2.1.2 gcc/6.4.0 mvapich2/2.3 intel/19.0.4 intel-mpi/2019.4 intel/18.0.2 mvapich2/2.2 intel/19.0.4 openmpi/2.1.2 friendly-testing hdf5-parallel/1.8.16 pnetcdf/1.11.2 netcdf-h5parallel/4.7.3 mkl/2019.0.4 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld $ENV{PNETCDF_PATH} $ENV{NETCDF_PATH} $ENV{MKLROOT} romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable LANL Linux Cluster, 36 pes/node, batch system slurm ba-fe.*.lanl.gov LINUX intel,gnu openmpi,impi,mvapich climateacme /lustre/scratch4/turquoise/$ENV{USER}/E3SM/scratch /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/atm/datm7 /lustre/scratch4/turquoise/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER /turquoise/usr/projects/climate/SHARED_CLIMATE/software/wolf/cprnc/v0.40/cprnc 4 e3sm_developer slurm e3sm 36 32 TRUE srun -n {{ total_tasks }} /usr/share/Modules/init/perl.pm /usr/share/Modules/init/python.py /etc/profile.d/z00_lmod.sh /etc/profile.d/z00_lmod.csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module module cmake/3.16.2 gcc/6.4.0 openmpi/2.1.2 gcc/6.4.0 mvapich2/2.3 intel/19.0.4 intel-mpi/2019.4 intel/18.0.2 mvapich2/2.2 intel/19.0.4 openmpi/2.1.2 friendly-testing hdf5-parallel/1.8.16 pnetcdf/1.11.2 netcdf-h5parallel/4.7.3 mkl/2019.0.4 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld $ENV{PNETCDF_PATH} $ENV{NETCDF_PATH} $ENV{MKLROOT} romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable Chicoma CPU-only nodes at LANL IC. Each node has 2 AMD EPYC 7H12 64-Core (Milan) 512GB ch-fe* Linux gnu,nvidia,intel,aocc,amdclang mpich /lustre/scratch4/turquoise/$ENV{USER}/E3SM/scratch/chicoma-cpu /usr/projects/e3sm/inputdata /usr/projects/e3sm/inputdata/atm/datm7 /lustre/scratch4/turquoise/$ENV{USER}/E3SM/archive/$CASE /lustre/scratch4/turquoise/$ENV{USER}/E3SM/input_data/ccsm_baselines/$COMPILER /usr/projects/climate/SHARED_CLIMATE/software/badger/cprnc 10 e3sm_developer 4 slurm e3sm 256 64 TRUE srun --label -n {{ total_tasks }} -N {{ num_nodes }} -c $ENV{OMP_NUM_THREADS} $SHELL{if [ 128 -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -m plane=$SHELL{echo `./xmlquery --value MAX_MPITASKS_PER_NODE`} /usr/share/lmod/8.3.1/init/perl /usr/share/lmod/8.3.1/init/python /usr/share/lmod/8.3.1/init/sh /usr/share/lmod/8.3.1/init/csh /usr/share/lmod/lmod/libexec/lmod perl /usr/share/lmod/lmod/libexec/lmod python module module cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf PrgEnv-gnu PrgEnv-nvidia PrgEnv-cray PrgEnv-aocc craype-accel-nvidia80 craype-accel-host PrgEnv-gnu/8.3.3 gcc/12.1.0 PrgEnv-nvidia/8.3.3 nvidia/22.3 PrgEnv-intel/8.3.3 intel-classic/2022.2.1 PrgEnv-aocc aocc/3.2.0 PrgEnv-aocc aocc/3.2.0 craype-accel-host cray-libsci craype cray-mpich/8.1.21 libfabric/1.15.0.0 cray-hdf5-parallel/1.12.2.1 cray-netcdf-hdf5parallel/4.9.0.1 cray-parallel-netcdf/1.12.3.1 cmake/3.20.3 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1 1 128M spread threads FALSE /usr/projects/climate/SHARED_CLIMATE/software/chicoma-cpu/perl5-only-switch/lib/perl5 romio_ds_write=disable;romio_ds_read=disable;romio_cb_write=enable;romio_cb_read=enable software MPI_Bcast -1 Mesabi batch queue LINUX intel openmpi /home/reichpb/scratch /home/reichpb/shared/cesm_inputdata /home/reichpb/shared/cesm_inputdata/atm/datm7 USERDEFINED_optional_run USERDEFINED_optional_run/$COMPILER USERDEFINED_optional_test 2 pbs chen1718 at umn dot edu 24 24 TRUE aprun -n {{ total_tasks }} -S {{ tasks_per_numa }} -N $MAX_MPITASKS_PER_NODE -d $ENV{OMP_NUM_THREADS} $CASEROOT/run $CASEROOT/exedir Itasca batch queue LINUX intel openmpi /home/reichpb/scratch /home/reichpb/shared/cesm_inputdata /home/reichpb/shared/cesm_inputdata/atm/datm7 USERDEFINED_optional_run USERDEFINED_optional_run/$COMPILER USERDEFINED_optional_test 2 pbs chen1718 at umn dot edu 8 8 aprun -n {{ total_tasks }} -S {{ tasks_per_numa }} -N $MAX_MPITASKS_PER_NODE -d $ENV{OMP_NUM_THREADS} $CASEROOT/run $CASEROOT/exedir Lawrencium LR3 cluster at LBL, OS is Linux (intel), batch system is SLURM n000* LINUX intel,gnu openmpi ac_acme /global/scratch/$ENV{USER} /global/scratch/$ENV{USER}/cesm_input_datasets/ /global/scratch/$ENV{USER}/cesm_input_datasets/atm/datm7 $CIME_OUTPUT_ROOT/cesm_archive/$CASE $CIME_OUTPUT_ROOT/cesm_baselines/$COMPILER /$CIME_OUTPUT_ROOT/cesm_tools/cprnc/cprnc 4 slurm rgknox and glemieux at lbl dot gov 8 8 TRUE mpirun -np {{ total_tasks }} -npernode $MAX_MPITASKS_PER_NODE mpirun -np {{ total_tasks }} -npernode $MAX_MPITASKS_PER_NODE /etc/profile.d/modules.sh /etc/profile.d/modules.csh /usr/Modules/init/perl.pm /usr/Modules/python.py module module /usr/Modules/bin/modulecmd perl /usr/Modules/bin/modulecmd python cmake/3.15.0 perl xml-libxml/2.0116 python/3.6 intel/2016.4.072 mkl netcdf/4.4.1.1-intel-s openmpi netcdf/4.4.1.1-intel-p gcc/6.3.0 lapack/3.8.0-gcc netcdf/5.4.1.1-gcc-s openmpi/2.0.2-gcc openmpi/3.0.1-gcc netcdf/4.4.1.1-gcc-p openmpi/2.0.2-gcc $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld Lawrencium LR6 cluster at LBL, OS is Linux (intel), batch system is SLURM n000* LINUX intel,gnu openmpi ac_acme /global/scratch/$ENV{USER} /global/scratch/$ENV{USER}/cesm_input_datasets/ /global/scratch/$ENV{USER}/cesm_input_datasets/atm/datm7 $CIME_OUTPUT_ROOT/cesm_archive/$CASE $CIME_OUTPUT_ROOT/cesm_baselines/$COMPILER /$CIME_OUTPUT_ROOT/cesm_tools/cprnc/cprnc 4 slurm rgknox and glemieux at lbl dot gov 32 32 TRUE mpirun -np {{ total_tasks }} mpirun -np {{ total_tasks }} /etc/profile.d/modules.sh /etc/profile.d/modules.csh /usr/Modules/init/perl.pm /usr/Modules/python.py module module /usr/Modules/bin/modulecmd perl /usr/Modules/bin/modulecmd python cmake/3.15.0 perl xml-libxml/2.0116 python/3.6 intel/2016.4.072 mkl netcdf/4.4.1.1-intel-s openmpi netcdf/4.4.1.1-intel-p gcc/6.3.0 lapack/3.8.0-gcc netcdf/5.4.1.1-gcc-s openmpi/2.0.2-gcc openmpi/3.0.1-gcc netcdf/4.4.1.1-gcc-p openmpi/2.0.2-gcc $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld FATES development machine at LBNL, System76 Thelio Massive Workstation Pop!_OS 20.04 lobata LINUX gnu openmpi $ENV{HOME}/scratch/ /data/cesmdataroot/inputdata /data/cesmdataroot/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE $ENV{HOME}/scratch/ctsm-baselines /home/glemieux/Repos/cime/tools/cprnc/cprnc make 16 none glemieux at lbl dot gov 4 4 FALSE mpirun -np {{ total_tasks }} --map-by ppr:{{ tasks_per_node }}:socket:PE=$ENV{OMP_NUM_THREADS} --bind-to hwthread /usr/share/modules/init/python.py /usr/share/modules/init/perl.pm /usr/share/modules/init/sh /usr/share/modules/init/csh /usr/bin/modulecmd python /usr/bin/modulecmd perl module module hdf5 netcdf-c netcdf-fortran esmf small developer workhorse at lbl climate sciences LINUX gnu openmpi ngeet /raid1/lbleco/e3sm/ /home/rgknox/Models/InputDatasets/cesm_input_data/ /home/rgknox/Models/InputDatasets/cesm_input_data/atm/datm7/ /home/rgknox/Models//cesm_archive/$CASE /home/rgknox/Models//cesm_baselines/$COMPILER /raid1/lbleco/cesm/cesm_tools/cprnc/cprnc 1 none rgknox at lbl gov 16 16 FALSE mpirun -np {{ total_tasks }} -npernode $MAX_MPITASKS_PER_NODE /usr/local/share/cmake-3.21/ ORNL Summit. Node: 2x POWER9 + 6x Volta V100, 22 cores/socket, 4 HW threads/core. .*summit.* LINUX ibm,ibmgpu,pgi,pgigpu,gnu,gnugpu spectrum-mpi cli115 cli115 /gpfs/alpine/proj-shared/cli115 .* /gpfs/alpine/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch /gpfs/alpine/cli115/world-shared/e3sm/inputdata /gpfs/alpine/cli115/world-shared/e3sm/inputdata/atm/datm7 /gpfs/alpine/$PROJECT/proj-shared/$ENV{USER}/archive/$CASE /gpfs/alpine/cli115/world-shared/e3sm/baselines/$COMPILER /gpfs/alpine/cli115/world-shared/e3sm/tools/cprnc.summit/cprnc 8 e3sm_developer 4 lsf e3sm 84 18 42 42 84 18 42 42 TRUE jsrun -X 1 $SHELL{if [ {{ total_tasks }} -eq 1 ];then echo --nrs 1 --rs_per_host 1;else echo --nrs $NUM_RS --rs_per_host $RS_PER_NODE;fi} --tasks_per_rs $SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc} -d plane:$SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc} --cpu_per_rs $ENV{CPU_PER_RS} --gpu_per_rs $ENV{GPU_PER_RS} --bind packed:smt:$ENV{OMP_NUM_THREADS} --latency_priority $ENV{LTC_PRT} --stdio_mode prepended $ENV{JSRUN_THREAD_VARS} $ENV{SMPIARGS} /sw/summit/lmod/8.4/init/sh /sw/summit/lmod/8.4/init/csh /sw/summit/lmod/8.4/init/env_modules_python.py /sw/summit/lmod/8.4/init/perl module /sw/summit/lmod/8.4/libexec/lmod python module module DefApps python/3.7-anaconda3 subversion/1.14.0 git/2.31.1 cmake/3.20.2 essl/6.3.0 netlib-lapack/3.8.0 nvhpc/21.11 xl/16.1.1-10 gcc/9.1.0 cuda/10.1.243 cuda/11.0.3 cuda/10.1.243 spectrum-mpi/10.4.0.3-20210112 hdf5/1.10.7 netcdf-c/4.8.0 netcdf-fortran/4.4.5 parallel-netcdf/1.12.2 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 $ENV{OLCF_NETCDF_C_ROOT} $ENV{OLCF_NETCDF_FORTRAN_ROOT} $ENV{OLCF_ESSL_ROOT} $ENV{OLCF_HDF5_ROOT} $ENV{OLCF_PARALLEL_NETCDF_ROOT} 0 -E OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} -E OMP_PROC_BIND=spread -E OMP_PLACES=threads -E OMP_STACKSIZE=256M --smpiargs="-gpu" 2 21 0 cpu-cpu $SHELL{echo "2*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} $SHELL{echo "(`./xmlquery --value MAX_TASKS_PER_NODE`+41)/42"|bc} 2 21 0 cpu-cpu $SHELL{echo "2*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} $SHELL{echo "(`./xmlquery --value MAX_TASKS_PER_NODE`+41)/42"|bc} 1 1 mlx5_3:1,mlx5_0:1 mlx5_0:1,mlx5_3:1 2 21 0 cpu-cpu $SHELL{echo "2*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} $SHELL{echo "(`./xmlquery --value MAX_TASKS_PER_NODE`+41)/42"|bc} 6 7 1 gpu-cpu $SHELL{echo "6*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} 6 3 1 gpu-cpu $SHELL{echo "6*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} 6 7 1 gpu-cpu $SHELL{echo "6*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} 1 1 mlx5_3:1,mlx5_0:1 mlx5_0:1,mlx5_3:1 /gpfs/alpine/cli115/world-shared/3rdparty/adios2/2.8.3.patch/spectrum-mpi-10.4.0.3/xl-16.1.1-10 /gpfs/alpine/cli115/world-shared/3rdparty/adios2/2.8.3.patch/spectrum-mpi-10.4.0.3/nvhpc-21.11 /gpfs/alpine/cli115/world-shared/3rdparty/adios2/2.8.3.patch/spectrum-mpi-10.4.0.3/gcc-9.1.0 ORNL Ascent. Node: 2x POWER9 + 6x Volta V100, 22 cores/socket, 4 HW threads/core. .*ascent.* LINUX ibm,ibmgpu,pgi,pgigpu,gnu,gnugpu spectrum-mpi cli115 cli115 /gpfs/wolf/proj-shared/$PROJECT cli115 /gpfs/wolf/$PROJECT/proj-shared/$ENV{USER}/e3sm_scratch /gpfs/wolf/cli115/world-shared/e3sm/inputdata /gpfs/wolf/cli115/world-shared/e3sm/inputdata/atm/datm7 /gpfs/wolf/$PROJECT/proj-shared/$ENV{USER}/archive/$CASE /gpfs/wolf/cli115/world-shared/e3sm/baselines/$COMPILER /gpfs/wolf/cli115/world-shared/e3sm/tools/cprnc/cprnc 8 e3sm_integration 4 lsf e3sm 84 18 42 42 84 18 42 42 TRUE jsrun -X 1 $SHELL{if [ {{ total_tasks }} -eq 1 ];then echo --nrs 1 --rs_per_host 1;else echo --nrs $NUM_RS --rs_per_host $RS_PER_NODE;fi} --tasks_per_rs $SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc} -d plane:$SHELL{echo "({{ tasks_per_node }} + $RS_PER_NODE - 1)/$RS_PER_NODE"|bc} --cpu_per_rs $ENV{CPU_PER_RS} --gpu_per_rs $ENV{GPU_PER_RS} --bind packed:smt:$ENV{OMP_NUM_THREADS} --latency_priority $ENV{LTC_PRT} --stdio_mode prepended $ENV{JSRUN_THREAD_VARS} $ENV{SMPIARGS} /sw/ascent/lmod/lmod/init/sh /sw/ascent/lmod/lmod/init/csh /sw/ascent/lmod/init/env_modules_python.py /sw/ascent/lmod/lmod/libexec/lmod python module module DefApps python/3.8-anaconda3 git/2.31.1 subversion cmake/3.22.2 essl/6.3.0 netlib-lapack/3.9.1 nvhpc/21.11 cuda/10.1.243 xl/16.1.1-10 cuda/11.0.3 gcc/9.1.0 spectrum-mpi/10.4.0.3-20210112 hdf5/1.10.7 netcdf-c/4.8.1 netcdf-fortran/4.4.5 parallel-netcdf/1.12.2 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 9000 /gpfs/wolf/cli115/world-shared/e3sm/soft/perl/5.26.0/bin:$ENV{PATH} $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf_version))} $ENV{OLCF_ESSL_ROOT} $ENV{OLCF_HDF5_ROOT} 0 -E OMP_NUM_THREADS=$ENV{OMP_NUM_THREADS} -E OMP_PROC_BIND=spread -E OMP_PLACES=threads -E OMP_STACKSIZE=256M --smpiargs="-gpu" 2 21 0 cpu-cpu $SHELL{echo "2*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} $SHELL{echo "(`./xmlquery --value MAX_TASKS_PER_NODE`+41)/42"|bc} 6 7 1 gpu-cpu $SHELL{echo "6*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} 6 3 1 gpu-cpu $SHELL{echo "6*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} pgc++ 6 7 1 gpu-cpu $SHELL{echo "6*((`./xmlquery --value TOTAL_TASKS` + `./xmlquery --value TASKS_PER_NODE` - 1)/`./xmlquery --value TASKS_PER_NODE`)"|bc} Medium sized linux cluster at BNL, torque scheduler. LINUX gnu openmpi,mpi-serial /data/$ENV{USER} /data/Model_Data/cesm_input_datasets/ /data/Model_Data/cesm_input_datasets/atm/datm7 $CIME_OUTPUT_ROOT/cesm_archive/$CASE $CIME_OUTPUT_ROOT/cesm_baselines /data/software/cesm_tools/cprnc/cprnc 4 pbs sserbin@bnl.gov 12 12 12 FALSE mpirun -np {{ total_tasks }} -npernode $MAX_TASKS_PER_NODE /etc/profile.d/modules.sh /etc/profile.d/modules.csh /usr/share/Modules/init/perl.pm /usr/share/Modules/init/python.py module module /usr/bin/modulecmd perl /usr/bin/modulecmd python perl/5.22.1 libxml2/2.9.2 maui/3.3.1 python/2.7.15 python/3.6.2 gcc/5.4.0 gfortran/5.4.0 hdf5/1.8.19fates netcdf/4.4.1.1-gnu540-fates openmpi/2.1.1-gnu540 openmpi/2.1.1-gnu540 /data/software/hdf5/1.8.19fates /data/software/netcdf/4.4.1.1-gnu540-fates ORNL experimental/evaluation cluster tulip.* LINUX gnu openmpi /home/groups/coegroup/e3sm/scratch/$USER /home/groups/coegroup/e3sm/inputdata2 /home/groups/coegroup/e3sm/inputdata2/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /home/groups/coegroup/e3sm/baselines/$COMPILER /home/groups/coegroup/e3sm/tools/cprnc/cprnc 16 e3sm_developer 4 slurm e3sm 64 32 FALSE mpirun --tag-output -n {{ total_tasks }} --map-by ppr:1:core:PE=$ENV{OMP_NUM_THREADS} --bind-to core /cm/local/apps/environment-modules/current/init/python /cm/local/apps/environment-modules/current/init/sh /cm/local/apps/environment-modules/current/init/csh /cm/local/apps/environment-modules/current/bin/modulecmd python module module gcc cce PrgEnv-cray cray-mvapich2 cmake/3.17.0 /home/users/twhite/share/modulefiles svn/1.10.6 gcc/8.1.0 blas/gcc/64/3.8.0 lapack/gcc/64/3.8.0 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld /home/groups/coegroup/e3sm/soft/perl5/lib/perl5 /home/groups/coegroup/e3sm/soft/netcdf/4.4.1c-4.2cxx-4.4.4f/gcc/8.2.0 gcc g++ gfortran /home/groups/coegroup/e3sm/soft/openmpi/2.1.6/gcc/8.2.0/bin:$ENV{PATH} /home/groups/coegroup/e3sm/soft/openmpi/2.1.6/gcc/8.2.0/lib:/home/groups/coegroup/e3sm/soft/netcdf/4.4.1c-4.2cxx-4.4.4f/gcc/8.2.0/lib:$ENV{LD_LIBRARY_PATH} /home/groups/coegroup/e3sm/soft/pnetcdf/1.12.1/gcc/8.2.0/openmpi/2.1.6 128M threads Google Cloud cluster using compute nodes c2d-compute-112's gcpe3sm12 gcpe3sm12* LINUX gnu openmpi /home/$USER/e3sm/scratch /home/inputdata /home/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /home/baselines/$COMPILER /home/tools/cprnc/cprnc 20 e3sm_developer 8 slurm e3sm 112 56 FALSE srun --mpi=pmi2 --label -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit -c $SHELL{echo `./xmlquery --value MAX_TASKS_PER_NODE`/ {{ tasks_per_node }} |bc} $SHELL{if [ `./xmlquery --value MAX_TASKS_PER_NODE` -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -m plane={{ tasks_per_node }} /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod python module module /opt/apps/spack/share/spack/modules/linux-centos7-zen2 /opt/apps/spack/share/spack/modules/linux-centos7-x86_64_v3 gcc openmpi binutils netlib-lapack openblas hdf5 netcdf-c parallel-netcdf gcc/12.2.0 openmpi/4.1.4 cmake perl perl-xml-libxml netcdf-c netcdf-cxx netcdf-fortran parallel-netcdf hdf5 netlib-lapack openblas $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.2 0.20 $SHELL{dirname $(dirname $(which h5diff))} $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf-config))} /opt/apps/spack/opt/spack/linux-centos7-zen2/gcc-12.2.0/openblas-0.3.21-z66r7lyxwkhsshgreexm4cedffp73scp /opt/apps/spack/opt/spack/linux-centos7-zen2/gcc-12.2.0/netlib-lapack-3.10.1-lkhddpuidlw2z74g5ui6eq5iattsfjxp $ENV{PERL5LIB}:/opt/apps/spack/opt/spack/linux-centos7-zen2/gcc-12.2.0/perl-5.36.0-sly2pft2edg2p3iyijfyy6dzntusokno/lib/site_perl/5.36.0 FALSE 128M threads Google Cloud cluster with c2-compute-60's gcp-e3sm10 gcp-e3sm10* LINUX gnu openmpi /home/$USER/e3sm/scratch /home/inputdata /home/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /home/baselines/$COMPILER /home/tools/cprnc/cprnc 16 e3sm_developer 4 slurm e3sm 60 30 FALSE srun --mpi=pmi2 --label -n {{ total_tasks }} -N {{ num_nodes }} --kill-on-bad-exit -c $SHELL{echo `./xmlquery --value MAX_TASKS_PER_NODE`/ {{ tasks_per_node }} |bc} $SHELL{if [ `./xmlquery --value MAX_TASKS_PER_NODE` -ge `./xmlquery --value MAX_MPITASKS_PER_NODE` ]; then echo "--cpu_bind=cores"; else echo "--cpu_bind=threads";fi;} -m plane={{ tasks_per_node }} /usr/share/lmod/lmod/init/env_modules_python.py /usr/share/lmod/lmod/init/sh /usr/share/lmod/lmod/init/csh /usr/share/lmod/lmod/libexec/lmod python module module /apps/spack/share/spack/modules/linux-centos7-cascadelake gcc openmpi gcc/12.2.0 openmpi-gcc@12.2.0 cmake perl perl-xml-libxml netcdf-c-gcc@12.2.0 netcdf-cxx-gcc@12.2.0 netcdf-fortran-gcc@12.2.0 parallel-netcdf-gcc@12.2.0 hdf5-gcc@12.2.0 netlib-lapack-gcc@12.2.0 openblas-gcc@12.2.0 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.2 0.20 $SHELL{dirname $(dirname $(which h5diff))} $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf-config))} /apps/spack/opt/spack/linux-centos7-cascadelake/gcc-12.2.0/openblas-0.3.20-nxcsxdi56nj2gxyo65iyuaecp3cbd4xd /apps/spack/opt/spack/linux-centos7-cascadelake/gcc-12.2.0/netlib-lapack-3.10.1-xjw3q4abrpdihbyvx72em7l4wrzxm3zp FALSE 128M threads RIKEN-CCS Fugaku: Fujitsu A64FX 48 cores/node. fn01sv.* LINUX gnu,fj fujitsu hp210190 /data/hp210190/ .* /data/hp210190/$USER/scratch /data/hp210190/inputdata /data/hp210190/inputdata/atm/datm7 /data/hp210190/$USER/scratch/archive/$CASE /data/hp210190/baselines/$COMPILER /data/hp210190/tools/cprnc/cprnc 8 e3sm_integration 4 moab E3SM 48 48 FALSE mpirun -n {{ total_tasks }} -std e3sm.log.$LID /vol0003/hp210190/data/soft/spack-v0.17.0/share/spack/setup-env.sh /vol0003/hp210190/data/soft/spack-v0.17.0/share/spack/setup-env.csh spack spack --all gcc @11.2.0%gcc@8.4.1 arch=linux-rhel8-a64fx fujitsu-mpi @head%gcc@11.2.0 arch=linux-rhel8-a64fx --loaded;ln -sf /lib64/libhwloc.so.15 /tmp/libhwloc.so.5 --all netcdf-c @4.8.1 %fj@4.7.0 arch=linux-rhel8-a64fx netcdf-cxx @4.2 %fj@4.7.0 arch=linux-rhel8-a64fx netcdf-fortran @4.5.3 %fj@4.7.0 arch=linux-rhel8-a64fx parallel-netcdf@1.12.2%fj@4.7.0 arch=linux-rhel8-a64fx netlib-lapack @3.9.1 %fj@4.7.0 arch=linux-rhel8-a64fx --loaded $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1000 /data/hp210190/soft/perl5/lib/perl5:/home/hp210190/u02380/perl5/lib/perl5 share_band /data/hp210190/soft/netcdf/4.4.1c-4.2cxx-4.4.4f/gcc8.3.1 /data/hp210190/soft/spack-v0.16/opt/spack/linux-rhel8-a64fx/gcc-8.3.1/netlib-lapack-3.8.0-jhmofiqoky6ajxmda5caawfhqnrirmm5 /tmp:/data/hp210190/soft/spack-v0.16/opt/spack/linux-rhel8-a64fx/gcc-8.3.1/netlib-lapack-3.8.0-jhmofiqoky6ajxmda5caawfhqnrirmm5/lib64:$ENV{LD_LIBRARY_PATH} $SHELL{dirname $(dirname $(which nc-config))} $SHELL{dirname $(dirname $(which nf-config))} $SHELL{dirname $(dirname $(which pnetcdf_version))} 128M cores ERDC XC40, os is CNL, 44 pes/node, batch system is PBS onyx CNL intel mpich NPSCA07935242 /p/app/unsupported/RASM/acme .* $ENV{WORKDIR} /p/app/unsupported/RASM/acme/inputdata /p/app/unsupported/RASM/acme/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /p/app/unsupported/RASM/acme/baselines/$COMPILER /p/app/unsupported/RASM/tools/cprnc/cprnc 8 e3sm_developer pbs rasm 44 44 aprun -n {{ total_tasks }} -N $SHELL{if [ `./xmlquery --value MAX_MPITASKS_PER_NODE` -gt `./xmlquery --value TOTAL_TASKS` ];then echo `./xmlquery --value TOTAL_TASKS`;else echo `./xmlquery --value MAX_MPITASKS_PER_NODE`;fi;} --cc depth -d $SHELL{echo `./xmlquery --value MAX_TASKS_PER_NODE`/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} -j $SHELL{if [ 64 -ge `./xmlquery --value MAX_TASKS_PER_NODE` ];then echo 1;else echo `./xmlquery --value MAX_TASKS_PER_NODE`/64|bc;fi;} /opt/modules/default/init/perl.pm /opt/modules/default/init/python.py /opt/modules/default/init/sh /opt/modules/default/init/csh /opt/modules/default/bin/modulecmd perl /opt/modules/default/bin/modulecmd python module module PrgEnv-intel PrgEnv-cray PrgEnv-gnu PrgEnv-pgi intel cce gcc cray-parallel-netcdf cray-parallel-hdf5 pmi cray-libsci cray-mpich2 cray-mpich cray-netcdf cray-hdf5 cray-netcdf-hdf5parallel craype-mic-knl craype-sandybridge craype-ivybridge craype papi cray-petsc cray-libsci esmf craype PrgEnv-intel/6.0.9 intel intel/19.1.3.304 cray-mpich cray-mpich/7.7.16 cray-hdf5 cray-hdf5-parallel cray-netcdf-hdf5parallel cray-parallel-netcdf netcdf cray-netcdf/4.7.4.0 cray-hdf5/1.12.0.0 cray-parallel-netcdf/1.12.1.0 cray-libsci cmake/intel-19.1.3.304/3.21.0 cray-libsci/20.09.1 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1 1 /opt/cray/pe/netcdf/4.7.4.0/intel/19.1 64M spread threads FALSE yes yes NavyDSRC Cray EX, os is CNL, 128 pes/node, batch system is PBS narwhal CNL intel mpich NPSCA07935242 /p/work1/projects/RASM/acme .* $ENV{WORKDIR} /p/work1/projects/RASM/acme/inputdata /p/work1/projects/RASM/acme/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /p/work1/projects/RASM/acme/baselines/$COMPILER /p/work1/projects/RASM/tools/cprnc/cprnc 8 e3sm_developer pbs e3sm 128 128 TRUE aprun -n {{ total_tasks }} -N $SHELL{if [ `./xmlquery --value MAX_MPITASKS_PER_NODE` -gt `./xmlquery --value TOTAL_TASKS` ];then echo `./xmlquery --value TOTAL_TASKS`;else echo `./xmlquery --value MAX_MPITASKS_PER_NODE`;fi;} --cc depth -d $SHELL{echo `./xmlquery --value MAX_TASKS_PER_NODE`/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} -j $SHELL{if [ 64 -ge `./xmlquery --value MAX_TASKS_PER_NODE` ];then echo 1;else echo `./xmlquery --value MAX_TASKS_PER_NODE`/64|bc;fi;} /opt/cray/pe/modules/3.2.11.5/init/perl.pm /opt/cray/pe/modules/3.2.11.5/init/python.py /opt/cray/pe/modules/3.2.11.5/init/sh /opt/cray/pe/modules/3.2.11.5/init/csh /opt/cray/pe/modules/3.2.11.5/bin/modulecmd perl /opt/cray/pe/modules/3.2.11.5/bin/modulecmd python module module PrgEnv-intel PrgEnv-cray PrgEnv-gnu PrgEnv-nvidia PrgEnv-aocc intel cray-mpich cray-hdf5 cray-hdf5-parallel cray-netcdf cray-netcdf-hdf5parallel cray-parallel-netcdf PrgEnv-intel/8.0.0 intel intel-classic intel-classic/2021.3.0 cray-mpich cray-mpich/8.1.14 cray-netcdf/4.7.4.7 cray-hdf5/1.12.0.7 cray-parallel-netcdf/1.12.1.7 cray-libsci cray-libsci/21.08.1.2 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1 1 64M spread threads FALSE yes yes AFRL Cray EX, os is CNL, 128 pes/node, batch system is PBS warhawk CNL intel mpich NPSCA07935242 /p/work1/projects/RASM/acme .* $ENV{WORKDIR} /p/work1/projects/RASM/acme/inputdata /p/work1/projects/RASM/acme/inputdata/atm/datm7 $CIME_OUTPUT_ROOT/archive/$CASE /p/work1/projects/RASM/acme/baselines/$COMPILER /p/work1/projects/RASM/tools/cprnc/cprnc 8 e3sm_developer pbs rasm 128 128 aprun -n {{ total_tasks }} -N $SHELL{if [ `./xmlquery --value MAX_MPITASKS_PER_NODE` -gt `./xmlquery --value TOTAL_TASKS` ];then echo `./xmlquery --value TOTAL_TASKS`;else echo `./xmlquery --value MAX_MPITASKS_PER_NODE`;fi;} --cc depth -d $SHELL{echo `./xmlquery --value MAX_TASKS_PER_NODE`/`./xmlquery --value MAX_MPITASKS_PER_NODE`|bc} -j $SHELL{if [ 64 -ge `./xmlquery --value MAX_TASKS_PER_NODE` ];then echo 1;else echo `./xmlquery --value MAX_TASKS_PER_NODE`/64|bc;fi;} /p/app/Modules/4.7.1/init/perl.pm /p/app/Modules/4.7.1/init/python.py /p/app/Modules/4.7.1/init/sh /p/app/Modules/4.7.1/init/csh /p/app/Modules/4.7.1/bin/modulecmd perl /p/app/Modules/4.7.1/bin/modulecmd python module module PrgEnv-intel PrgEnv-cray PrgEnv-gnu PrgEnv-nvidia craype-x86-rome craype-network-ofi cray-dsmml perftools-base cray-libsci cce intel gcc cray-mpich cray-hdf5 cray-hdf5-parallel cray-netcdf cray-netcdf-hdf5parallel cray-parallel-netcdf PrgEnv-intel/8.0.0 intel intel-classic/2021.3.0 cray-mpich cray-mpich/8.1.9 cray-pals/1.0.17 cray-netcdf/4.7.4.4 cray-hdf5/1.12.0.4 cray-parallel-netcdf/1.12.1.4 cray-libsci cmake/3.21.4 cray-libsci/21.08.1.2 $CIME_OUTPUT_ROOT/$CASE/run $CIME_OUTPUT_ROOT/$CASE/bld 0.1 1 1 64M spread threads FALSE yes yes Docker docker LINUX gnu,gnuX openmpi CIME /storage/timings CIME /storage/cases /storage/inputdata /storage/inputdata-clmforc /storage/archive/$CASE /storage/baselines/$COMPILER /storage/tools/cprnc make 8 e3sm_developer none scream 8 8 mpiexec -n {{ total_tasks }} --oversubscribe $CASEROOT/run $CASEROOT/bld 1 1 ${EXEROOT}/e3sm.exe >> e3sm.log.$LID 2>&1