Run PINACLES on Perlmutter

Batch script example

This test case script and json input files are available in the shared directory ($runscriptdir in the batch script). Make sure to edit the path to your own scratch directory and also the email address.

sub_test_1km_share.sh

#!/bin/bash

#SBATCH -N 8
#SBATCH -C cpu
#SBATCH -q debug
#SBATCH -t 00:10:00
#SBATCH -J test_1km_share
#SBATCH -A m1867
#SBATCH --mail-user=user.email@org.gov
#SBATCH --mail-type=FAIL,END

module load python
module load cray-fftw

module list

conda activate /global/common/software/m1867/python/pinacles_share
pinacle_dir="/global/common/software/m1867/PINACLES"

#need to edit the following variables ------
simname="test_1km_share" #I usually use this as part of the json file name

runscriptdir="/global/cfs/cdirs/wcm_code/PINACLES/share/runscripts"

inputfile="${simname}.json"
#--------------------------------------

# 8 nodes, 128 tasks/node
srun -n 1024 -c 2 --cpu_bind=cores python -u ${pinacle_dir}/main_simulation.py ${runscriptdir}/${inputfile}

Accompanied json input file

test_1km_share.json

{
    "damping": {
        "depth": 10000.0,
        "timescale": 60.0,
        "vars": [
            "w"
        ]
    },
    "fields": {
        "frequency": 1800.0,
        "io_type": "zarr"
    },
    "grid": {
        "l": [
            600000.0,
            600000.0,
            33000.0
        ],
        "n": [
            600,
            600,
            165
        ],
        "n_halo": [
            4,
            4,
            4
        ]
    },
    "lbc": {
        "open_boundary_treatment": "nest",
        "type": "periodic"
    },
    "meta": {
        "casename": "rce",
        "output_directory": "/pscratch/sd/where you want to run}",
        "random_seed": 1,
        "simname": "test_1km_shared"
    },
    "microphysics": {
        "nccnst": 100000000.0,
        "scheme": "p3"
    },
    "momentum_advection": {
        "type": "weno5_z"
    },
    "radiation": {
        "compute_radiation": true,
        "input_filepath": "./testbed_inputs/ena_25Jan_inputs.nc",
        "rrtmg_lib_path": "/global/common/software/m1867/PINACLES/pinacles/externals/rrtmg_wrapper",
        "type": "rce",
        "update_frequency": 300.0
    },
    "restart": {
        "infile": "path  to previous simulation output",
        "restart_simulation": false
    },
    "scalar_advection": {
        "type": "weno5_z"
    },
    "sgs": {
        "model": "smagorinsky",
        "smagorinsky": {
            "cs": 0.17,
            "prt": 0.3333333333333333
        }
    },
    "stats": {
        "frequency": 600.0,
        "modules": []
    },
    "time": {
        "cfl": 0.8,
        "time_max": 3600
    },
    "towers": {
        "frequency": 600.0,
        "location": []
    }
}