if (NOT DEFINED PROJECT_NAME) cmake_minimum_required(VERSION 3.3) cmake_policy(SET CMP0057 NEW) set(SCREAM_CIME_BUILD FALSE) else() set(SCREAM_CIME_BUILD TRUE) endif() if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0") # The new behavior for cmp0074 makes cmake use (rather than ignore) # any _ROOT env/cmake variable previously set. cmake_policy(SET CMP0074 NEW) endif() set (EAMXX_VERSION_MAJOR 1) set (EAMXX_VERSION_MINOR 0) set (EAMXX_VERSION_PATCH 0) if ($ENV{SCREAM_FORCE_CONFIG_FAIL}) message(FATAL_ERROR "Failed, as instructed by environment") endif() # Add the ./cmake folder to cmake path. Also add EKAT's cmake folder set (EKAT_CMAKE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/../../externals/ekat/cmake) list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake ${CMAKE_CURRENT_SOURCE_DIR}/../cmake/modules ${EKAT_CMAKE_PATH} ${EKAT_CMAKE_PATH}/tpls ) if (SCREAM_CIME_BUILD) list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_SOURCE_DIR}/cmake/cime) endif () if (Kokkos_ENABLE_CUDA) if (Kokkos_ENABLE_CUDA_RELOCATABLE_DEVICE_CODE) if (Kokkos_ENABLE_DEBUG_BOUNDS_CHECK) string (CONCAT msg "Kokkos_ENALBE_CUDA_RELOCATABLE_DEVICE_CODE=ON, and Kokkos_ENALBE_DEBUG_BOUNDS_CHECK=ON.\n" " -> Disabling bounds checks, to prevent internal compiler errors.") message(WARNING "${msg}") set (Kokkos_ENABLE_DEBUG_BOUNDS_CHECK OFF CACHE BOOL "" FORCE) else() string (CONCAT msg "Kokkos_ENALBE_CUDA_RELOCATABLE_DEVICE_CODE=ON.\n" " -> Disabling bounds checks, to prevent internal compiler errors.") message(STATUS "${msg}") set (Kokkos_ENABLE_DEBUG_BOUNDS_CHECK OFF CACHE BOOL "" FORCE) endif() endif() endif() # Homme's composef90 library (built for f90 support) requires Kokkos::Serial # to be defined, which in turn requires the CMake var Kokkos_ENABLE_SERIAL # to be on. For now, simply ensure Kokkos Serial is enabled option (Kokkos_ENABLE_SERIAL "" ON) # We want to use C++17 in EAMxx set(CMAKE_CXX_STANDARD 17) if (NOT SCREAM_CIME_BUILD) project(SCREAM CXX C Fortran) if (SCREAM_CORI_HACK) list(APPEND CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "ifcore") list(REMOVE_ITEM CMAKE_CXX_IMPLICIT_LINK_LIBRARIES "ifport") endif() else() # Ensure our languages are all enabled enable_language(C CXX Fortran) endif() # Print the sha of the last commit (useful to double check which version was tested on CDash) execute_process (COMMAND git rev-parse HEAD WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} OUTPUT_VARIABLE LAST_GIT_COMMIT_SHA OUTPUT_STRIP_TRAILING_WHITESPACE) set(EAMXX_GIT_VERSION ${LAST_GIT_COMMIT_SHA} CACHE STRING "The sha of the last git commit.") message(STATUS "The sha of the last commit is ${EAMXX_GIT_VERSION}") set(SCREAM_DOUBLE_PRECISION TRUE CACHE BOOL "Set to double precision (default True)") # Set the scream base and src directory, to be used across subfolders set(SCREAM_BASE_DIR ${CMAKE_CURRENT_SOURCE_DIR}) set(SCREAM_SRC_DIR ${CMAKE_CURRENT_SOURCE_DIR}/src) set(SCREAM_BIN_DIR ${CMAKE_CURRENT_BINARY_DIR}) # Shortcut function, to print a variable function (print_var var) message ("${var}: ${${var}}") endfunction () function (check_pack_size master_pack_size pack_size name) math (EXPR PACK_MODULO "${master_pack_size} % ${pack_size}") if ((pack_size GREATER master_pack_size) OR (NOT PACK_MODULO EQUAL 0)) message (FATAL_ERROR "Invalid '${name}' size of ${pack_size}. Needs to be <= ${master_pack_size} and be a factor of it") endif() endfunction () # Compute reasonable defaults. This needs to happen before the CACHE variables # are set. string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_ci) if (CMAKE_BUILD_TYPE_ci STREQUAL "debug") set (SCREAM_DEBUG TRUE) else () set (SCREAM_DEBUG FALSE) endif() # Add RRTMGP debug checks. Note, we might consider also adding RRTMGP_EXPENSIVE_CHECKS # to turn on the RRTMGP internal checks here as well, via # option (RRTMGP_EXPENSIVE_CHECKS "Turn on internal RRTMGP error checking" ${SCREAM_DEBUG}) # and then adding to scream_config.h: # #cmakedefine RRTMGP_EXPENSIVE_CHECKS option (SCREAM_RRTMGP_DEBUG "Turn on extra debug checks in RRTMGP" ${SCREAM_DEBUG}) enable_testing() include(CTest) set (EAMXX_ENABLE_GPU FALSE CACHE BOOL "") set (CUDA_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there? set (HIP_BUILD FALSE CACHE BOOL "") #needed for yakl if kokkos vars are not visible there? # Determine if this is a Cuda build. if (Kokkos_ENABLE_CUDA) # Add CUDA as a language for CUDA builds enable_language(CUDA) set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE) set (CUDA_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there? endif () # Determine if this is a Cuda build. if (Kokkos_ENABLE_HIP) # Add CUDA as a language for CUDA builds enable_language(HIP) set (EAMXX_ENABLE_GPU TRUE CACHE BOOL "" FORCE) set (HIP_BUILD TRUE CACHE BOOL "" FORCE) #needed for yakl if kokkos vars are not visible there? endif () if( NOT "${CMAKE_CXX_COMPILER_ID}" MATCHES "[Cc]lang" ) set (CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -cpp") endif() ### Scream default configuration options set(DEFAULT_MAX_RANKS 4) set(DEFAULT_MAX_THREADS 16) set(DEFAULT_MIMIC_GPU FALSE) set(DEFAULT_FPE FALSE) set(DEFAULT_PACK_SIZE 16) set(DEFAULT_POSSIBLY_NO_PACK FALSE) if (EAMXX_ENABLE_GPU) # On the GPU, the pack size must be 1 set(DEFAULT_PACK_SIZE 1) set(DEFAULT_MAX_THREADS 1) # Limit to 1 rank, cause parallel builds testing might limit the number of available gpus set(DEFAULT_MAX_RANKS 1) else() if (SCREAM_DEBUG) set(DEFAULT_MIMIC_GPU TRUE) endif() endif () set(DEFAULT_FPMODEL "precise") set(DEFAULT_LIB_ONLY FALSE) if (SCREAM_CIME_BUILD) set(DEFAULT_LIB_ONLY TRUE) endif() set(DEFAULT_NUM_VERTICAL_LEV 72) find_path(NF_CONFIG_SEARCH nf-config) if (NF_CONFIG_SEARCH) execute_process(COMMAND ${NF_CONFIG_SEARCH}/nf-config --prefix RESULT_VARIABLE NF_STATUS OUTPUT_VARIABLE NF_CONFIG_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE ) if (NF_STATUS EQUAL 0) set(DEFAULT_NetCDF_Fortran_PATH ${NF_CONFIG_OUTPUT}) endif() endif() find_path(NC_CONFIG_SEARCH nc-config) if (NC_CONFIG_SEARCH) execute_process(COMMAND ${NC_CONFIG_SEARCH}/nc-config --prefix RESULT_VARIABLE NC_STATUS OUTPUT_VARIABLE NC_CONFIG_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE ) if (NC_STATUS EQUAL 0) set(DEFAULT_NetCDF_C_PATH ${NC_CONFIG_OUTPUT}) endif() endif() if (DEFINED ENV{SCREAM_MACHINE}) set(DEFAULT_SCREAM_MACHINE $ENV{SCREAM_MACHINE}) elseif(MACH) # MACH comes from CIME build system set(DEFAULT_SCREAM_MACHINE ${MACH}) endif() set(DEFAULT_SMALL_KERNELS FALSE) if (Kokkos_ENABLE_HIP) set(DEFAULT_SMALL_KERNELS TRUE) endif() ### Set CACHE vars set(SCREAM_MIMIC_GPU ${DEFAULT_MIMIC_GPU} CACHE BOOL "Mimic GPU to correctness-test inter-column parallelism on non-GPU platform") set(SCREAM_PACK_CHECK_BOUNDS FALSE CACHE BOOL "If defined, scream::pack objects check indices against bounds") set(SCREAM_TEST_DATA_DIR ${CMAKE_CURRENT_BINARY_DIR}/data CACHE PATH "Location of data files generated by tests") set(SCREAM_LIB_ONLY ${DEFAULT_LIB_ONLY} CACHE BOOL "Only build libraries, no exes") set(NetCDF_Fortran_PATH ${DEFAULT_NetCDF_Fortran_PATH} CACHE FILEPATH "Path to netcdf fortran installation") set(NetCDF_C_PATH ${DEFAULT_NetCDF_C_PATH} CACHE FILEPATH "Path to netcdf C installation") set(SCREAM_MACHINE ${DEFAULT_SCREAM_MACHINE} CACHE STRING "The CIME/SCREAM name for the current machine") option(SCREAM_MPI_ON_DEVICE "Whether to use device pointers for MPI calls" ON) option(SCREAM_ENABLE_MAM "Whether to enable MAM aerosol support" OFF) set(SCREAM_SMALL_KERNELS ${DEFAULT_SMALL_KERNELS} CACHE STRING "Use small, non-monolothic kokkos kernels") if (NOT SCREAM_SMALL_KERNELS) set(EKAT_DISABLE_WORKSPACE_SHARING TRUE CACHE STRING "") endif() # For now, only used in share/grid/remap/refining_remapper_rma.*pp option (EAMXX_ENABLE_EXPERIMENTAL_CODE "Compile one-sided MPI for refining remappers" OFF) # Handle input root if (SCREAM_MACHINE AND NOT SCREAM_INPUT_ROOT) execute_process(COMMAND ${CMAKE_CURRENT_SOURCE_DIR}/scripts/query-cime ${SCREAM_MACHINE} DIN_LOC_ROOT RESULT_VARIABLE QC_STATUS OUTPUT_VARIABLE QC_OUTPUT OUTPUT_STRIP_TRAILING_WHITESPACE ) if (QC_STATUS EQUAL 0) set(DEFAULT_SCREAM_INPUT_ROOT ${QC_OUTPUT}) endif() endif() set(SCREAM_INPUT_ROOT ${DEFAULT_SCREAM_INPUT_ROOT} CACHE PATH "Root of downloaded input files. Should match DIN_LOC_ROOT on CIME machines") if (NOT SCREAM_INPUT_ROOT) string (CONCAT msg " -> Value not found for SCREAM_INPUT_ROOT.\n" " You have three ways for setting this folder. In order of precedence:\n" " - define an env var SCREAM_MACHINE (must be a CIME-supported machine)\n" " - define a cmake cache entry SCREAM_MACHINE (must be a CIME-supported machine)\n" " - define a cmake cache entry SCREAM_INPUT_ROOT.\n" " Ultimately, SCREAM_INPUT_ROOT is the folder where input data will be downloaded.\n") message ("${msg}") message (FATAL_ERROR "ERROR! Aborting...") elseif(NOT IS_DIRECTORY ${SCREAM_INPUT_ROOT}) string (CONCAT msg " -> SCREAM_INPUT_ROOT=${SCREAM_INPUT_ROOT} is not a directory." " You have three ways for setting this folder. In order of precedence:\n" " - define an env var SCREAM_MACHINE (must be a CIME-supported machine)\n" " - define a cmake cache entry SCREAM_MACHINE (must be a CIME-supported machine)\n" " - define a cmake cache entry SCREAM_INPUT_ROOT.\n" " Ultimately, SCREAM_INPUT_ROOT is the folder where input data will be downloaded.\n") message ("${msg}") message (FATAL_ERROR "ERROR! Aborting...") else() execute_process(COMMAND test -w ${SCREAM_INPUT_ROOT} RESULT_VARIABLE res) if (NOT res EQUAL 0) string (CONCAT msg " -> No write permissions on ${SCREAM_INPUT_ROOT}\n" " If some input files need to be downloaded, this may result in an error." " NOTE: You have three ways for setting this folder. In order of precedence:\n" " - define an env var SCREAM_MACHINE (must be a CIME-supported machine)\n" " - define a cmake cache entry SCREAM_MACHINE (must be a CIME-supported machine)\n" " - define a cmake cache entry SCREAM_INPUT_ROOT.\n" " Ultimately, SCREAM_INPUT_ROOT is the folder where input data will be downloaded.\n") message ("${msg}") endif() endif() set (SCREAM_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/scream CACHE PATH "" FORCE) set (TOPO_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/cam/topo CACHE PATH "" FORCE) set (IOP_DATA_DIR ${SCREAM_INPUT_ROOT}/atm/cam/scam/iop CACHE PATH "" FORCE) # # Handle test level # # Constants set(SCREAM_TEST_LEVEL_AT "0") set(SCREAM_TEST_LEVEL_NIGHTLY "1") set(SCREAM_TEST_LEVEL_EXPERIMENTAL "2") set(SCREAM_TEST_LEVEL "AT" CACHE STRING "The test level to run. Default is AT. NIGHTLY will run additional tests but is not guaranteed to PASS. EXPERIMENTAL will run even more tests with failures being more likely") if (SCREAM_TEST_LEVEL STREQUAL "AT") set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_AT}) elseif (SCREAM_TEST_LEVEL STREQUAL "NIGHTLY") set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_NIGHTLY}) elseif (SCREAM_TEST_LEVEL STREQUAL "EXPERIMENTAL") set(SCREAM_TEST_LEVEL ${SCREAM_TEST_LEVEL_EXPERIMENTAL}) else() message(FATAL_ERROR "Unknown SCREAM_TEST_LEVEL '${SCREAM_TEST_LEVEL}'") endif() # Assuming SCREAM_LIB_ONLY is FALSE (else, no exec is built at all), we provide the option # of building only baseline-related execs. By default, this option is off (menaing "build everything"). # However, when generating baselines, this can be useful to reduce the amount of stuff compiled. set(SCREAM_BASELINES_ONLY FALSE CACHE BOOL "Whether building only baselines-related executables") # Certain builds are not meant to compare against baselines. For instance, a valgrind build # has the sole purpose of detecting memory errors. For these builds, we can disable baselines tests set(SCREAM_ENABLE_BASELINE_TESTS TRUE CACHE BOOL "Whether to run baselines-related tests") if (SCREAM_BASELINES_ONLY AND NOT SCREAM_ENABLE_BASELINE_TESTS) message (FATAL_ERROR "Makes no sense to set SCREAM_BASELINES_ONLY=ON,\n" "but set SCREAM_ENABLE_BASELINE_TESTS=OFF.") endif() # Set number of vertical levels set(SCREAM_NUM_VERTICAL_LEV ${DEFAULT_NUM_VERTICAL_LEV} CACHE STRING "The number of levels used in the vertical grid." ) option(SCREAM_HAS_LEAP_YEAR "Whether scream uses leap years or not" ON) ## Work out pack sizes. # Determine the master pack size. set(SCREAM_PACK_SIZE ${DEFAULT_PACK_SIZE} CACHE STRING "The number of scalars in a scream::pack::Pack and Mask. Larger packs have good performance on conditional-free loops due to improved caching.") # With the master pack size determined, we have constraints on the others. set(DEFAULT_SMALL_PACK_SIZE ${SCREAM_PACK_SIZE}) # For some routines, SKX may have better performance with pksize=1 if (Kokkos_ARCH_SKX) set(DEFAULT_POSSIBLY_NO_PACK TRUE) endif () set(SCREAM_SMALL_PACK_SIZE ${DEFAULT_SMALL_PACK_SIZE} CACHE STRING "The number of scalars in a scream::pack::SmallPack and SmallMask. Smaller packs can have better performance in loops with conditionals since more of the packs will have masks with uniform value.") set(SCREAM_POSSIBLY_NO_PACK ${DEFAULT_POSSIBLY_NO_PACK} CACHE BOOL "Set possibly-no-pack to this value. You can set it to something else to restore packs on SKX for testing.") set (DEFAULT_POSSIBLY_NO_PACK_SIZE ${SCREAM_PACK_SIZE}) if (SCREAM_POSSIBLY_NO_PACK) set (DEFAULT_POSSIBLY_NO_PACK_SIZE 1) endif () set (SCREAM_POSSIBLY_NO_PACK_SIZE ${DEFAULT_POSSIBLY_NO_PACK_SIZE}) # Checks on pack sizes relative to the master one: check_pack_size(${SCREAM_PACK_SIZE} ${SCREAM_SMALL_PACK_SIZE} "small pack") # This one is an internal check, as the user cannot set SCREAM_POSSIBLY_NO_PACK_SIZE now. check_pack_size(${SCREAM_PACK_SIZE} ${SCREAM_POSSIBLY_NO_PACK_SIZE} "possibly no pack") ## Now we have pack sizes. Proceed with other config options that depend on ## these. if (SCREAM_DEBUG) set(DEFAULT_FPMODEL "strict") if (${SCREAM_PACK_SIZE} EQUAL 1 AND NOT ${EAMXX_ENABLE_GPU}) set(DEFAULT_FPE TRUE) endif () endif() set(SCREAM_FPMODEL ${DEFAULT_FPMODEL} CACHE STRING "Compiler floating point model") set(SCREAM_FPE ${DEFAULT_FPE} CACHE BOOL "Enable floating point error exception") ###