#!/bin/bash -fe ###################################################################### # This E3SMv2 run script is modified to: # 1. Read emission data in native SE-grid format # (Ref. https://github.com/eagles-project/E3SM-Private/pull/27) # 2. Produce SE-grid emission data offline by conservative remapping # using ggen (Ref. https://github.com/TaufiqHassan/ggen) # Original RLL-grid high res. data directory: # /compyfs/inputdata/atm/cam/chem/trop_mozart_aero/emis/DECK_ne120/ # 3. Use ERA5 nudging data # (Ref. https://github.com/TaufiqHassan/ERA5_IPT) ###################################################################### main() { # For debugging, uncomment libe below #set -x # --- Configuration flags ---- # Machine and project readonly MACHINE=pm-cpu readonly PROJECT="m3525" # esmd # Simulation readonly COMPSET="F20TR" readonly RESOLUTION="ne30pg2_EC30to60E2r2" # ne30pg2 resolution #readonly RESOLUTION="northamericax4v1pg2_WC14to60E2r3" # NA RRM resolution readonly CASE_NAME="AS_Total" readonly CASE_GROUP="v2.LR" # Code and compilation readonly CHECKOUT="E3SM" readonly BRANCH="TaufiqHassan/maint-2.0-eagles-aerosols-native" # Branch with code modification to read SE data readonly CHERRY=( ) readonly DEBUG_COMPILE=false # PE layout readonly NTASKS=1024 # 30 nodes (16 tasks in each) readonly NTHRDS=1 # Run options readonly MODEL_START_TYPE="initial" # 'initial', 'continue', 'branch', 'hybrid' readonly START_DATE="1999-01-01" # Set paths readonly ROOT_DIR="/pscratch/sd/h/hass877" readonly CODE_ROOT="${ROOT_DIR}/${CHECKOUT}" readonly CASE_ROOT="${ROOT_DIR}/${CHECKOUT}/cases/${CASE_NAME}" readonly CRUN_ROOT="${ROOT_DIR}/e3sm_scratch/${CASE_NAME}" # Sub-directories readonly CASE_BUILD_DIR=${CRUN_ROOT}/build readonly CASE_ARCHIVE_DIR=${CRUN_ROOT}/archive readonly SOURCEMOD_DIR=${CASE_ROOT}/tests/L_1x1_nmonths/case_scripts/SourceMods # Modified codes will be copied here readonly MODS_DIR=/pscratch/sd/h/hass877/inputs # Modified codes directory readonly run='L_1x1_nmonths' if [ "${run}" != "production" ]; then # Short test simulations tmp=($(echo $run | tr "_" " ")) layout=${tmp[0]} units=${tmp[2]} resubmit=$(( ${tmp[1]%%x*} -1 )) length=${tmp[1]##*x} readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/tests/${run}/case_scripts readonly CASE_RUN_DIR=${CASE_ROOT}/tests/${run}/run readonly PELAYOUT=${layout} readonly WALLTIME="00:15:00" readonly STOP_OPTION=${units} readonly STOP_N=${length} readonly REST_OPTION=${STOP_OPTION} readonly REST_N=3 readonly RESUBMIT=${resubmit} readonly DO_SHORT_TERM_ARCHIVING=false else # Production simulation readonly CASE_SCRIPTS_DIR=${CASE_ROOT}/case_scripts readonly CASE_RUN_DIR=${CASE_ROOT}/run readonly PELAYOUT="L" readonly WALLTIME="34:00:00" readonly STOP_OPTION="nyears" readonly STOP_N="50" readonly REST_OPTION="nyears" readonly REST_N="5" readonly RESUBMIT="9" readonly DO_SHORT_TERM_ARCHIVING=false fi # Coupler history readonly HIST_OPTION="nyears" readonly HIST_N="5" # Leave empty (unless you understand what it does) readonly OLD_EXECUTABLE="" # --- Toggle flags for what to do ---- do_fetch_code=false do_create_newcase=true do_case_setup=true do_case_build=true do_case_submit=false do_cp_mods=true # Modified code to SourceMods # --- Now, do the work --- # Make directories created by this script world-readable umask 022 # Fetch code from Github fetch_code # Create case create_newcase # Copy mods cp_mods # Setup case_setup # Build case_build # Configure runtime options runtime_options # Copy script into case_script directory for provenance copy_script # Submit case_submit # All done echo $'\n----- All done -----\n' } # ====================================================================================== # Custom user_nl settings # namelist variables list: # https://github.com/kaizhangpnl/kaizhangpnl.github.io/blob/master/source/output_h0.csv # ====================================================================================== user_nl() { cat << EOF >> user_nl_eam inithist = 'YEARLY' inithist_all = .true. ncdata = '/pscratch/sd/h/hass877/inputs/AppStudyEmis_ne30pg2.eam.i.1850-01-01-00000.nc' !!ncdata = '/compyfs/hass877/IPTtool/input/F20TR_v2.northamericax4v1pg2.eam.i.2010-01-01-00000.nc' !! NA RRM nhtfrq = 0,-24,-1,-24,-24 mfilt = 1,1,24,1,1 avgflag_pertape = 'A','A','I','L','L' lcltod_start = 0,0, 0, 36000, 46800 !! 10am, 1pm lcltod_stop = 0,0,0, 39600, 50400 !! 11am, 2pm fincl1 = 'PS', 'WSUBI', !! characteristic vertical velocity for ice nucleation 'WLARGE', !! large scale vertical velocity 'WSIG', !! subgrid scale vertical velocity 'WSUBI2', !! Mean updraft, with stddev=f(TKE) 'DCST', !! dcs 'RHICE', !! RH ice 'TOT_CLD_VISTAU', 'TOT_ICLD_VISTAU', 'LIQ_ICLD_VISTAU', 'ICE_ICLD_VISTAU', 'SNOW_ICLD_VISTAU', 'AODVIS', !! AeroCom indirect diagnostics 'angstrm', 'cod', 'cdr', 'cdnc', 'cdnum', 'icnum', 'clt', 'lcc', 'lwp', 'iwp', 'icc', 'icnc', 'icr', 'LHFLX', 'SHFLX', 'OMEGA500', 'rh700', 'colrv', 'ccn', 'ccn.1bl', 'ccn.3bl', 'ptop', 'ttop', 'rwp', 'lwp2', 'iwp2', 'autoconv', 'accretn', 'FSUTOA_d1', 'FSUTOAC_d1', 'FSUTOA', 'FSUTOAC', 'FLUTC', 'FLUT', 'PRECC', 'PRECL', 'PRECT', 'TH7001000', !!=========Other options=========!! docosp = .true., cosp_lite = .true., cosp_ncolumns = 10 cosp_nradsteps = 3 cosp_lmisr_sim = .true. cosp_lisccp_sim = .true. cosp_lmodis_sim = .true. cosp_llidar_sim = .true. history_amwg = .true. history_aero_optics = .true. history_aerosol = .true. history_clubb = .true. history_budget = .true. history_verbose = .true. hist_hetfrz_classnuc = .true. do_aerocom_ind3 = .true. rad_diag_1 = 'A:Q:H2O', 'N:O2:O2', 'N:CO2:CO2', 'A:O3:O3', 'N:N2O:N2O', 'N:CH4:CH4', 'N:CFC11:CFC11', 'N:CFC12:CFC12', !....................................................... ! emissions ! Feeding the native SE-grid data here !....................................................... read_se_emis = .true. ext_frc_specifier = 'SO2 -> /pscratch/sd/h/hass877/AppStudy/so2_1999-2020_E3SM_elev_emis_ne30pg2.nc', 'SOAG -> /pscratch/sd/h/hass877/inputs/new_emission_files/cmip6_mam4_soag_elev_1999-2019_c20191108_RLL384x576_ne30pg2.nc', 'bc_a4 -> /pscratch/sd/h/hass877/AppStudy/bc_a4_1999-2020_E3SM_elev_emis_ne30pg2.nc', 'num_a1 -> /pscratch/sd/h/hass877/AppStudy/num_a1_1999-2020_E3SM_elev_emis_ne30pg2.nc', 'num_a2 -> /pscratch/sd/h/hass877/AppStudy/num_a2_1999-2020_E3SM_elev_emis_ne30pg2.nc', 'num_a4 -> /pscratch/sd/h/hass877/AppStudy/num_a4_1999-2020_E3SM_elev_emis_ne30pg2.nc', 'pom_a4 -> /pscratch/sd/h/hass877/AppStudy/pom_a4_1999-2020_E3SM_elev_emis_ne30pg2.nc', 'so4_a1 -> /pscratch/sd/h/hass877/AppStudy/so4_a1_1999-2020_E3SM_elev_emis_ne30pg2.nc', 'so4_a2 -> /pscratch/sd/h/hass877/AppStudy/so4_a2_1999-2020_E3SM_elev_emis_ne30pg2.nc' ext_frc_type = 'INTERP_MISSING_MONTHS' srf_emis_specifier = 'DMS -> /global/cfs/cdirs/e3sm/inputdata/atm/cam/chem/trop_mozart_aero/emis/DMSflux.1850-2100.1deg_latlon_conserv.POPmonthlyClimFromACES4BGC_c20160727.nc', 'SO2 -> /pscratch/sd/h/hass877/AppStudy/so2_1999-2020_E3SM_surf_emis_ne30pg2.nc', 'bc_a4 -> /pscratch/sd/h/hass877/AppStudy/bc_a4_1999-2020_E3SM_surf_emis_ne30pg2.nc', 'num_a1 -> /pscratch/sd/h/hass877/AppStudy/num_a1_1999-2020_E3SM_surf_emis_ne30pg2.nc', 'num_a2 -> /pscratch/sd/h/hass877/AppStudy/num_a2_1999-2020_E3SM_surf_emis_ne30pg2.nc', 'num_a4 -> /pscratch/sd/h/hass877/AppStudy/num_a4_1999-2020_E3SM_surf_emis_ne30pg2.nc', 'pom_a4 -> /pscratch/sd/h/hass877/AppStudy/pom_a4_1999-2020_E3SM_surf_emis_ne30pg2.nc', 'so4_a1 -> /pscratch/sd/h/hass877/AppStudy/so4_a1_1999-2020_E3SM_surf_emis_ne30pg2.nc', 'so4_a2 -> /pscratch/sd/h/hass877/AppStudy/so4_a2_1999-2020_E3SM_surf_emis_ne30pg2.nc' srf_emis_type = 'INTERP_MISSING_MONTHS' !....................................................... ! fixed oxid forcing !....................................................... tracer_cnst_cycle_yr = 2015 tracer_cnst_datapath = '/pscratch/sd/h/hass877/inputs' tracer_cnst_file = 'oxid_1.9x2.5_L26_2015_c201010.nc' tracer_cnst_filelist = '' tracer_cnst_specifier = 'cnst_O3:O3','OH','NO3','HO2' tracer_cnst_type = 'CYCLICAL' !....................................................... ! fixed ghg forcing !....................................................... scenario_ghg = 'FIXED' ch4vmr = 1807.851e-9 n2ovmr = 323.141e-9 co2vmr = 312.821e-6 f11vmr = 768.7644e-12 f12vmr = 531.2820e-12 !....................................................... ! fixed linoz forcing !....................................................... linoz_data_cycle_yr = 2015 linoz_data_file = 'linoz1850-2015_2010JPL_CMIP6_10deg_58km_c20171109.nc' linoz_data_path = '/pscratch/sd/h/hass877/inputs' linoz_data_type = 'CYCLICAL' !....................................................... ! fixed volc forcing !....................................................... prescribed_volcaero_datapath = '/pscratch/sd/h/hass877/inputs' prescribed_volcaero_file = 'CMIP_DOE-ACME_radiation_average_1850-2014_v3_c20171204.nc' prescribed_volcaero_filetype = 'VOLC_CMIP6' prescribed_volcaero_type = 'CYCLICAL' prescribed_volcaero_cycle_yr = 1 !....................................................... ! fixed solar forcing !....................................................... solar_data_file = '/pscratch/sd/h/hass877/inputs/Solar_2010control_input4MIPS_c20181017.nc' solar_data_type = 'FIXED' solar_data_ymd = 20100101 solar_htng_spctrl_scl = .true. !....................................................... ! nudging !....................................................... Nudge_Model = .True. Nudge_Path = '/global/cfs/cdirs/m3525/pma/merra2/ne30pg2_72/' !!Nudge_Path = '/compyfs/hass877/IPTtool/output/NA_RRM/' !! NA RRM ERA5 data Nudge_File_Template = 'MERRA2_%y-%m-%d-%s.nc' !!Nudge_File_Template = 'ERA5_northamericax4v1pg2.%y-%m-%d-%s.nc' !! NA RRM ERA5 template Nudge_Times_Per_Day = 8 !! nudging input data frequency Model_Times_Per_Day = 48 !! 48 if dtime = 1800s !!Model_Times_Per_Day = 96 !! 96 if dtime = 900s (RRM uses highres dtime) !! Nuding profiles and nudging strength !! Nudge_Uprof = 2 !! 0 = off; 1 = CONSTANT; 2 = HEAVISIDE WINDOW FUNCTION (create window - 14 params) Nudge_Ucoef = 1. !! nudging strength as coefficient 0 < coef <= 1 Nudge_Vprof = 2 Nudge_Vcoef = 1. Nudge_Tprof = 0 Nudge_Tcoef = 0. Nudge_Qprof = 0 Nudge_Qcoef = 0. Nudge_PSprof = 0 Nudge_PScoef = 0. !! Prof end !! Nudge_Beg_Year = 1999 Nudge_Beg_Month = 1 Nudge_Beg_Day = 1 Nudge_End_Year = 2019 Nudge_End_Month = 12 Nudge_End_Day = 31 !! NA RRM horizontal window !! !!Nudge_Hwin_lat0 = 40.0 !!Nudge_Hwin_latWidth = 80. !!Nudge_Hwin_latDelta = 5. !!Nudge_Hwin_lon0 = 260. !!Nudge_Hwin_lonWidth = 120. !!Nudge_Hwin_lonDelta = 5. !! vert window Nudge_Vwin_Lindex = 6. !! vertical level index Nudge_Vwin_Hindex = 60. !! Nudging applied to Model.lev[Nudge_Vwin_Lindex:Nudge_Vwin_Hindex] Nudge_Vwin_Ldelta = 0.1 !! nudging window transition length Nudge_Vwin_Hdelta = 0.1 Nudge_Vwin_lo = 0. Nudge_Vwin_hi = 1. Nudge_Method = 'Linear' !! options: Step, Linear, and IMT (intermittent) Nudge_Loc_PhysOut = .True. Nudge_Tau = 6. !! relaxation time scale 3hr (Nudge_Times_Per_Day==8) Nudge_CurrentStep = .False. Nudge_File_Ntime = 1 !! time slice per file (1 per file for ERA5) EOF cat << EOF >> user_nl_elm finidat = '/pscratch/sd/h/hass877/inputs/AppStudyEmis_ne30pg2.elm.r.1850-01-01-00000.nc' !!finidat = '/compyfs/hass877/IPTtool/input/F20TR_v2.northamericax4v1pg2.elm.r.2010-01-01-00000.nc' !! NA RRM ! Override check_finidat_fsurdat_consistency = .false. EOF } patch_mpas_streams() { echo } ###################################################### ### Most users won't need to change anything below ### ###################################################### #----------------------------------------------------- fetch_code() { if [ "${do_fetch_code,,}" != "true" ]; then echo $'\n----- Skipping fetch_code -----\n' return fi echo $'\n----- Starting fetch_code -----\n' local path=${CODE_ROOT} local repo=e3sm echo "Cloning $repo repository branch $BRANCH under $path" if [ -d "${path}" ]; then echo "ERROR: Directory already exists. Not overwriting" exit 20 fi mkdir -p ${path} pushd ${path} # This will put repository, with all code git clone git@github.com:E3SM-Project/${repo}.git . # Setup git hooks rm -rf .git/hooks git clone git@github.com:E3SM-Project/E3SM-Hooks.git .git/hooks git config commit.template .git/hooks/commit.template # Check out desired branch git checkout ${BRANCH} # Custom addition if [ "${CHERRY}" != "" ]; then echo ----- WARNING: adding git cherry-pick ----- for commit in "${CHERRY[@]}" do echo ${commit} git cherry-pick ${commit} done echo ------------------------------------------- fi # Bring in all submodule components git submodule update --init --recursive popd } #----------------------------------------------------- create_newcase() { if [ "${do_create_newcase,,}" != "true" ]; then echo $'\n----- Skipping create_newcase -----\n' return fi echo $'\n----- Starting create_newcase -----\n' ${CODE_ROOT}/cime/scripts/create_newcase \ --case ${CASE_NAME} \ --case-group ${CASE_GROUP} \ --output-root ${CASE_ROOT} \ --script-root ${CASE_SCRIPTS_DIR} \ --handle-preexisting-dirs u \ --compset ${COMPSET} \ --res ${RESOLUTION} \ --machine ${MACHINE} \ --project ${PROJECT} \ --walltime ${WALLTIME} #\ #--pecount ${PELAYOUT} if [ $? != 0 ]; then echo $'\nNote: if create_newcase failed because sub-directory already exists:' echo $' * delete old case_script sub-directory' echo $' * or set do_newcase=false\n' exit 35 fi } #----------------------------------------------------- # Copy modified codes to SourceMods cp_mods() { if [ "${do_cp_mods,,}" != "true" ]; then echo $'\n----- Skipping cp_mods -----\n' return fi echo $'\n---- Copying modified codes to SourceMods ----\n' cp ${MODS_DIR}/modal_aero_amicphys.F90 ${SOURCEMOD_DIR}/src.eam/ } #----------------------------------------------------- case_setup() { if [ "${do_case_setup,,}" != "true" ]; then echo $'\n----- Skipping case_setup -----\n' return fi echo $'\n----- Starting case_setup -----\n' pushd ${CASE_SCRIPTS_DIR} # Setup some CIME directories ./xmlchange EXEROOT=${CASE_BUILD_DIR} ./xmlchange RUNDIR=${CASE_RUN_DIR} # Short term archiving ./xmlchange DOUT_S=${DO_SHORT_TERM_ARCHIVING^^} ./xmlchange DOUT_S_ROOT=${CASE_ARCHIVE_DIR} ./xmlchange NTASKS_ATM=$NTASKS ./xmlchange NTHRDS_ATM=$NTHRDS ./xmlchange ROOTPE_ATM='0' ./xmlchange NTASKS_LND=$NTASKS ./xmlchange NTHRDS_LND=$NTHRDS ./xmlchange ROOTPE_LND='0' ./xmlchange NTASKS_ROF=$NTASKS ./xmlchange NTHRDS_ROF=$NTHRDS ./xmlchange ROOTPE_ROF='0' ./xmlchange NTASKS_ICE=$NTASKS ./xmlchange NTHRDS_ICE=$NTHRDS ./xmlchange ROOTPE_ICE='0' ./xmlchange NTASKS_OCN=$NTASKS ./xmlchange NTHRDS_OCN=$NTHRDS ./xmlchange ROOTPE_OCN='0' ./xmlchange NTASKS_GLC=$NTASKS ./xmlchange NTHRDS_GLC=$NTHRDS ./xmlchange ROOTPE_GLC='0' ./xmlchange NTASKS_WAV=$NTASKS ./xmlchange NTHRDS_WAV=$NTHRDS ./xmlchange ROOTPE_WAV='0' ./xmlchange NTASKS_CPL=$(($NTASKS/8)) ./xmlchange NTHRDS_CPL=$NTHRDS ./xmlchange ROOTPE_CPL='0' # Build with COSP, except for a data atmosphere (datm) if [ `./xmlquery --value COMP_ATM` == "datm" ]; then echo $'\nThe specified configuration uses a data atmosphere, so cannot activate COSP simulator\n' else echo $'\nConfiguring E3SM to use the COSP simulator\n' ./xmlchange --id CAM_CONFIG_OPTS --append --val='-cosp' fi # Extracts input_data_dir in case it is needed for user edits to the namelist later local input_data_dir=`./xmlquery DIN_LOC_ROOT --value` # Custom user_nl user_nl # Finally, run CIME case.setup ./case.setup --reset popd } #----------------------------------------------------- case_build() { pushd ${CASE_SCRIPTS_DIR} # do_case_build = false if [ "${do_case_build,,}" != "true" ]; then echo $'\n----- case_build -----\n' if [ "${OLD_EXECUTABLE}" == "" ]; then # Ues previously built executable, make sure it exists if [ -x ${CASE_BUILD_DIR}/e3sm.exe ]; then echo 'Skipping build because $do_case_build = '${do_case_build} else echo 'ERROR: $do_case_build = '${do_case_build}' but no executable exists for this case.' exit 297 fi else # If absolute pathname exists and is executable, reuse pre-exiting executable if [ -x ${OLD_EXECUTABLE} ]; then echo 'Using $OLD_EXECUTABLE = '${OLD_EXECUTABLE} cp -fp ${OLD_EXECUTABLE} ${CASE_BUILD_DIR}/ else echo 'ERROR: $OLD_EXECUTABLE = '$OLD_EXECUTABLE' does not exist or is not an executable file.' exit 297 fi fi echo 'WARNING: Setting BUILD_COMPLETE = TRUE. This is a little risky, but trusting the user.' ./xmlchange BUILD_COMPLETE=TRUE # do_case_build = true else echo $'\n----- Starting case_build -----\n' # Turn on debug compilation option if requested if [ "${DEBUG_COMPILE^^}" == "TRUE" ]; then ./xmlchange DEBUG=${DEBUG_COMPILE^^} fi # Run CIME case.build ./case.build # Some user_nl settings won't be updated to *_in files under the run directory # Call preview_namelists to make sure *_in and user_nl files are consistent. ./preview_namelists fi popd } #----------------------------------------------------- runtime_options() { echo $'\n----- Starting runtime_options -----\n' pushd ${CASE_SCRIPTS_DIR} # Set simulation start date ./xmlchange RUN_STARTDATE=${START_DATE} # Segment length ./xmlchange STOP_OPTION=${STOP_OPTION,,},STOP_N=${STOP_N} # Restart frequency ./xmlchange REST_OPTION=${REST_OPTION,,},REST_N=${REST_N} # Coupler history ./xmlchange HIST_OPTION=${HIST_OPTION,,},HIST_N=${HIST_N} # Coupler budgets (always on) ./xmlchange BUDGETS=TRUE # Set resubmissions if (( RESUBMIT > 0 )); then ./xmlchange RESUBMIT=${RESUBMIT} fi # Run type # Start from default of user-specified initial conditions if [ "${MODEL_START_TYPE,,}" == "initial" ]; then ./xmlchange RUN_TYPE="startup" ./xmlchange CONTINUE_RUN="FALSE" # Continue existing run elif [ "${MODEL_START_TYPE,,}" == "continue" ]; then ./xmlchange CONTINUE_RUN="TRUE" elif [ "${MODEL_START_TYPE,,}" == "branch" ] || [ "${MODEL_START_TYPE,,}" == "hybrid" ]; then ./xmlchange RUN_TYPE=${MODEL_START_TYPE,,} ./xmlchange GET_REFCASE=${GET_REFCASE} ./xmlchange RUN_REFDIR=${RUN_REFDIR} ./xmlchange RUN_REFCASE=${RUN_REFCASE} ./xmlchange RUN_REFDATE=${RUN_REFDATE} echo 'Warning: $MODEL_START_TYPE = '${MODEL_START_TYPE} echo '$RUN_REFDIR = '${RUN_REFDIR} echo '$RUN_REFCASE = '${RUN_REFCASE} echo '$RUN_REFDATE = '${START_DATE} else echo 'ERROR: $MODEL_START_TYPE = '${MODEL_START_TYPE}' is unrecognized. Exiting.' exit 380 fi # Patch mpas streams files patch_mpas_streams popd } #----------------------------------------------------- case_submit() { if [ "${do_case_submit,,}" != "true" ]; then echo $'\n----- Skipping case_submit -----\n' return fi echo $'\n----- Starting case_submit -----\n' pushd ${CASE_SCRIPTS_DIR} # Run CIME case.submit ./case.submit popd } #----------------------------------------------------- copy_script() { echo $'\n----- Saving run script for provenance -----\n' local script_provenance_dir=${CASE_SCRIPTS_DIR}/run_script_provenance mkdir -p ${script_provenance_dir} local this_script_name=`basename $0` local script_provenance_name=${this_script_name}.`date +%Y%m%d-%H%M%S` cp -vp ${this_script_name} ${script_provenance_dir}/${script_provenance_name} } #----------------------------------------------------- # Silent versions of popd and pushd pushd() { command pushd "$@" > /dev/null } popd() { command popd "$@" > /dev/null } # Now, actually run the script #----------------------------------------------------- main