Replace cice5 with cice6 in ufs-s2s-model (#160)

* Replaces CICE5 with CICE6: Issue #159 

* Adds required  CICE6 items to nems.configure. Removes non-cmeps nems.configure files. Updates default_vars and edit_inputs to allow CICE6 resolution and processor count to be specified at run time. Updates ice_in_template for CICE6

* Updates to esmf8.1.0bs27 to allow ICE and WAV on separate PEs (Issue #172)

* Adds cpl_mode (Issue #138) and use_coldstart as new configurable variables (Issues #86, #53). 

* Fixes coldstart run sequence by calling the restart write phase before the accum avg phase (Issue #167)

.gitignore

@@ -0,0 +1,62 @@
+# Prerequisites
+*.d
+
+# Compiled Object files
+*.slo
+*.lo
+*.o
+*.obj
+
+# Precompiled Headers
+*.gch
+*.pch
+
+# Compiled Dynamic libraries
+*.so
+*.dylib
+*.dll
+
+# Fortran module files
+*.mod
+*.smod
+
+# Compiled Static libraries
+*.lai
+*.la
+*.a
+*.lib
+
+# Executables
+*.exe
+*.out
+*.app
+
+# system files
+*.swp
+*~
+.DS_Store
+
+# python files
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# repo-specific ignorables
+coupled*.mk
+
+# CMake files and directories
+build*/
+install*/
+CMakeLists.txt.user
+CMakeCache.txt
+CMakeFiles
+CMakeScripts
+cmake_install.cmake
+install_manifest.txt
+compile_commands.json
+CTestTestfile.cmake
+_deps

.gitmodules

@@ -6,10 +6,6 @@
 	path = NEMS
 	url = https://github.com/NOAA-EMC/NEMS.git
 	branch = develop
-[submodule "CICE_SRC"]
-	path = CICE_SRC
-	url = https://github.com/NOAA-EMC/CICE5.git
-	branch = develop
 [submodule "FMS"]
 	path = FMS
 	url = https://github.com/NOAA-GFDL/FMS
@@ -18,10 +14,6 @@
 	path = stochastic_physics
 	url = https://github.com/noaa-psd/stochastic_physics
 	branch = master
-[submodule "MOM6"]
-	path = MOM6
-	url = https://github.com/NOAA-EMC/MOM6-interface
-	branch = develop
 [submodule "CMEPS"]
 	path = CMEPS
 	url = https://github.com/NOAA-EMC/CMEPS.git
@@ -30,3 +22,15 @@
 	path = WW3
 	url = https://github.com/NOAA-EMC/WW3.git
 	branch = develop
+[submodule "CICE6"]
+	path = CICE-interface/CICE
+	url = https://github.com/NOAA-EMC/CICE.git
+	branch = emc/develop
+[submodule "MOM6-interface/mkmf"]
+	path = MOM6-interface/mkmf
+	url = https://github.com/NOAA-EMC/mkmf
+	branch = develop
+[submodule "MOM6-interface/MOM6"]
+	path = MOM6-interface/MOM6
+	url = https://github.com/NOAA-EMC/MOM6
+	branch = dev/emc

CICE-interface/CMakeLists.txt

@@ -0,0 +1,97 @@
+include(GNUInstallDirs)
+
+include("cice_files.cmake")
+
+# Configuration Options
+set(CICE_DRIVER "NUOPC" CACHE STRING "CICE OPTIONS: Choose CICE6 Driver." FORCE)
+set_property(CACHE CICE_DRIVER PROPERTY STRINGS "STANDALONE" "NUOPC" "DIRECT" "MCT")
+
+set(NUOPC_TARGET "CMEPS" CACHE STRING "CICE OPTIONS: Choose NUOPC implementation." FORCE)
+set_property(CACHE NUOPC_TARGET PROPERTY STRINGS "CMEPS" "DMI")
+
+message(STATUS "CICE: CICE_DRIVER = ${CICE_DRIVER}")
+message(STATUS "CICE: NUOPC_TARGET = ${NUOPC_TARGET}")
+
+# Choose files based on Serial or MPI build
+if(MPI_Fortran_FOUND)
+  message(STATUS "CICE: Build with MPI")
+  list(APPEND comm_list ${cice_mpi_comm_files})
+else()
+  message(STATUS "CICE: Build without MPI")
+  list(APPEND comm_list ${cice_serial_comm_files})
+endif()
+
+# Collect IO files for appropriate IO
+if(USE_NetCDF AND NetCDF_Fortran_FOUND)
+  message(STATUS "CICE: Build with NetCDF IO")
+  list(APPEND io_list ${cice_netcdf_io_files})
+elseif(USE_PIO AND PIO_Fortran_FOUND)
+  message(STATUS "CICE: Build with Parallel IO")
+  list(APPEND io_list ${cice_pio2_io_files})
+else()
+  message(STATUS "CICE: Build with Binary IO")
+  list(APPEND io_list ${cice_binary_io_files})
+endif()
+
+# Collect driver files
+# There are multiple drivers in the drivers area
+if(CICE_DRIVER STREQUAL "NUOPC")
+  if(NUOPC_TARGET STREQUAL "CMEPS")
+    list(APPEND driver_list ${cice_nuopc_cmeps_driver_files})
+  elseif(NUOPC_TARGET STREQUAL "DMI")
+    list(APPEND driver_list ${cice_nuopc_dmi_driver_files})
+  endif()
+elseif(CICE_DRIVER STREQUAL "DIRECT")
+  list(APPEND driver_list ${cice_direct_driver_files})
+elseif(CICE_DRIVER STREQUAL "MCT")
+  list(APPEND driver_list ${cice_mct_driver_files})
+else()
+  list(APPEND driver_list ${cice_standalone_driver_files})
+endif()
+
+# Merge all the lists together for the library
+list(APPEND lib_src_files
+  ${cice_shared_files}
+  ${icepack_files}
+  ${comm_list}
+  ${io_list}
+  ${driver_list}
+)
+
+set(libName "nuopccice")
+set(moduleDir "${CMAKE_CURRENT_BINARY_DIR}/include/${libName}")
+
+add_library(${libName} STATIC ${lib_src_files})
+add_library(${libName}::${libName} ALIAS ${libName})
+
+set_target_properties(${libName} PROPERTIES Fortran_MODULE_DIRECTORY
+                                            ${moduleDir})
+
+target_compile_definitions(${libName} PRIVATE -DESMF_VERSION_MAJOR=${ESMF_VERSION_MAJOR})
+target_include_directories(${libName} PRIVATE ${ESMF_MOD})
+
+if(NetCDF_Fortran_FOUND)
+  target_link_libraries(${libName} PUBLIC NetCDF::NetCDF_Fortran)
+endif()
+
+if(PIO_Fortran_FOUND)
+  target_link_libraries(${libName} PUBLIC PIO::PIO_Fortran)
+endif()
+
+target_include_directories(${libName} INTERFACE
+  $<BUILD_INTERFACE:${moduleDir}>
+  $<INSTALL_INTERFACE:include/${libName}>)
+
+if(OpenMP_Fortran_FOUND)
+  target_link_libraries(${libName} PUBLIC OpenMP::OpenMP_Fortran)
+endif()
+
+# Install compiled Fortran modules
+install(DIRECTORY ${moduleDir} DESTINATION ${CMAKE_INSTALL_INCLUDEDIR})
+
+# Install library
+install(
+  TARGETS ${libName}
+  LIBRARY DESTINATION lib
+  ARCHIVE DESTINATION lib
+  COMPONENT Library)

CICE-interface/cice_files.cmake

@@ -0,0 +1,185 @@
+list(APPEND cice_shared_files
+  #Shared List:
+  CICE/cicecore/shared/ice_arrays_column.F90
+  CICE/cicecore/shared/ice_calendar.F90
+  CICE/cicecore/shared/ice_constants.F90
+  CICE/cicecore/shared/ice_distribution.F90
+  CICE/cicecore/shared/ice_domain_size.F90
+  CICE/cicecore/shared/ice_fileunits.F90
+  CICE/cicecore/shared/ice_init_column.F90
+  CICE/cicecore/shared/ice_kinds_mod.F90
+  CICE/cicecore/shared/ice_restart_column.F90
+  CICE/cicecore/shared/ice_restart_shared.F90
+  CICE/cicecore/shared/ice_spacecurve.F90
+
+  #Analysis List:
+  CICE/cicecore/cicedynB/analysis/ice_diagnostics.F90
+  CICE/cicecore/cicedynB/analysis/ice_diagnostics_bgc.F90
+  CICE/cicecore/cicedynB/analysis/ice_history.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_bgc.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_drag.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_fsd.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_mechred.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_pond.F90
+  CICE/cicecore/cicedynB/analysis/ice_history_shared.F90
+
+  #Dynamics List:
+  CICE/cicecore/cicedynB/dynamics/ice_dyn_eap.F90
+  CICE/cicecore/cicedynB/dynamics/ice_dyn_evp.F90
+  CICE/cicecore/cicedynB/dynamics/ice_dyn_evp_1d.F90
+  CICE/cicecore/cicedynB/dynamics/ice_dyn_shared.F90
+  CICE/cicecore/cicedynB/dynamics/ice_transport_driver.F90
+  CICE/cicecore/cicedynB/dynamics/ice_transport_remap.F90
+
+  #General List:
+  CICE/cicecore/cicedynB/general/ice_flux.F90
+  CICE/cicecore/cicedynB/general/ice_flux_bgc.F90
+  CICE/cicecore/cicedynB/general/ice_forcing.F90
+  CICE/cicecore/cicedynB/general/ice_forcing_bgc.F90
+  CICE/cicecore/cicedynB/general/ice_init.F90
+  CICE/cicecore/cicedynB/general/ice_state.F90
+  CICE/cicecore/cicedynB/general/ice_step_mod.F90
+
+  #Infrastructure List
+  CICE/cicecore/cicedynB/infrastructure/ice_blocks.F90
+  CICE/cicecore/cicedynB/infrastructure/ice_domain.F90
+  CICE/cicecore/cicedynB/infrastructure/ice_grid.F90
+  CICE/cicecore/cicedynB/infrastructure/ice_read_write.F90
+  CICE/cicecore/cicedynB/infrastructure/ice_restart_driver.F90
+  CICE/cicecore/cicedynB/infrastructure/ice_restoring.F90
+)
+
+
+#Icepack List:
+list(APPEND icepack_files
+  CICE/icepack/columnphysics/icepack_aerosol.F90
+  CICE/icepack/columnphysics/icepack_age.F90
+  CICE/icepack/columnphysics/icepack_algae.F90
+  CICE/icepack/columnphysics/icepack_atmo.F90
+  CICE/icepack/columnphysics/icepack_brine.F90
+  CICE/icepack/columnphysics/icepack_firstyear.F90
+  CICE/icepack/columnphysics/icepack_flux.F90
+  CICE/icepack/columnphysics/icepack_fsd.F90
+  CICE/icepack/columnphysics/icepack_intfc.F90
+  CICE/icepack/columnphysics/icepack_isotope.F90
+  CICE/icepack/columnphysics/icepack_itd.F90
+  CICE/icepack/columnphysics/icepack_kinds.F90
+  CICE/icepack/columnphysics/icepack_mechred.F90
+  CICE/icepack/columnphysics/icepack_meltpond_cesm.F90
+  CICE/icepack/columnphysics/icepack_meltpond_lvl.F90
+  CICE/icepack/columnphysics/icepack_meltpond_topo.F90
+  CICE/icepack/columnphysics/icepack_mushy_physics.F90
+  CICE/icepack/columnphysics/icepack_ocean.F90
+  CICE/icepack/columnphysics/icepack_orbital.F90
+  CICE/icepack/columnphysics/icepack_parameters.F90
+  CICE/icepack/columnphysics/icepack_shortwave.F90
+  CICE/icepack/columnphysics/icepack_therm_0layer.F90
+  CICE/icepack/columnphysics/icepack_therm_bl99.F90
+  CICE/icepack/columnphysics/icepack_therm_itd.F90
+  CICE/icepack/columnphysics/icepack_therm_mushy.F90
+  CICE/icepack/columnphysics/icepack_therm_shared.F90
+  CICE/icepack/columnphysics/icepack_therm_vertical.F90
+  CICE/icepack/columnphysics/icepack_tracers.F90
+  CICE/icepack/columnphysics/icepack_warnings.F90
+  CICE/icepack/columnphysics/icepack_wavefracspec.F90
+  CICE/icepack/columnphysics/icepack_zbgc.F90
+  CICE/icepack/columnphysics/icepack_zbgc_shared.F90
+  CICE/icepack/columnphysics/icepack_zsalinity.F90
+)
+
+list(APPEND cice_shared_files_c
+  CICE/cicecore/cicedynB/infrastructure/ice_shr_reprosum86.c
+)
+
+#-- Using MPI
+list(APPEND cice_mpi_comm_files
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_boundary.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_broadcast.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_communicate.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_exit.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_gather_scatter.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_global_reductions.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_reprosum.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/mpi/ice_timers.F90
+)
+
+#-- Using Serial
+list(APPEND cice_serial_comm_files
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_boundary.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_broadcast.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_communicate.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_exit.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_gather_scatter.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_global_reductions.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_reprosum.F90
+  CICE/cicecore/cicedynB/infrastructure/comm/serial/ice_timers.F90
+)
+
+#-- Using binary IO
+list(APPEND cice_binary_io_files
+  CICE/cicecore/cicedynB/infrastructure/io/io_binary/ice_history_write.F90
+  CICE/cicecore/cicedynB/infrastructure/io/io_binary/ice_restart.F90
+)
+
+#-- Using NetCDF IO
+list(APPEND cice_netcdf_io_files
+  CICE/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_history_write.F90
+  CICE/cicecore/cicedynB/infrastructure/io/io_netcdf/ice_restart.F90
+)
+
+#PIO2 I/O List:
+list(APPEND cice_pio2_io_files
+  CICE/cicecore/cicedynB/infrastructure/io/io_pio2/ice_history_write.F90
+  CICE/cicecore/cicedynB/infrastructure/io/io_pio2/ice_pio.F90
+  CICE/cicecore/cicedynB/infrastructure/io/io_pio2/ice_restart.F90
+)
+
+#-- Using standalone driver
+list(APPEND cice_standalone_driver_files
+  CICE/cicecore/drivers/standalone/cice/CICE.F90
+  CICE/cicecore/drivers/standalone/cice/CICE_FinalMod.F90
+  CICE/cicecore/drivers/standalone/cice/CICE_InitMod.F90
+  CICE/cicecore/drivers/standalone/cice/CICE_RunMod.F90
+)
+
+#-- Using NUOPC CMEPS driver
+list(APPEND cice_nuopc_cmeps_driver_files
+  CICE/cicecore/drivers/nuopc/cmeps/CICE_FinalMod.F90
+  CICE/cicecore/drivers/nuopc/cmeps/CICE_InitMod.F90
+  CICE/cicecore/drivers/nuopc/cmeps/CICE_RunMod.F90
+  CICE/cicecore/drivers/nuopc/cmeps/cice_wrapper_mod.F90
+  CICE/cicecore/drivers/nuopc/cmeps/ice_comp_nuopc.F90
+  CICE/cicecore/drivers/nuopc/cmeps/ice_import_export.F90
+  CICE/cicecore/drivers/nuopc/cmeps/ice_prescribed_mod.F90
+  CICE/cicecore/drivers/nuopc/cmeps/ice_scam.F90
+  CICE/cicecore/drivers/nuopc/cmeps/ice_shr_methods.F90
+)
+
+#-- Using NUOPC DMI driver
+list(APPEND cice_nuopc_dmi_driver_files
+  CICE/cicecore/drivers/nuopc/dmi/CICE.F90
+  CICE/cicecore/drivers/nuopc/dmi/CICE_FinalMod.F90
+  CICE/cicecore/drivers/nuopc/dmi/CICE_InitMod.F90
+  CICE/cicecore/drivers/nuopc/dmi/CICE_RunMod.F90
+)
+
+#-- Using direct driver
+list(APPEND cice_direct_driver_files
+  CICE/cicecore/drivers/direct/hadgem3/CICE.F90
+  CICE/cicecore/drivers/direct/hadgem3/CICE_FinalMod.F90
+  CICE/cicecore/drivers/direct/hadgem3/CICE_InitMod.F90
+  CICE/cicecore/drivers/direct/hadgem3/CICE_RunMod.F90
+)
+
+#-- Using MCT driver
+list(APPEND cice_mct_driver_files
+  CICE/cicecore/drivers/mct/cesm1/CICE_FinalMod.F90
+  CICE/cicecore/drivers/mct/cesm1/CICE_InitMod.F90
+  CICE/cicecore/drivers/mct/cesm1/CICE_RunMod.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_comp_esmf.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_comp_mct.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_cpl_indices.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_import_export.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_prescribed_mod.F90
+  CICE/cicecore/drivers/mct/cesm1/ice_scam.F90
+)