Migrate to GW Python v3.12 environment on WCOSS2 (#4007)

This migrates the global workflow to the NCO-installed
global-workflow-specific Python environment on WCOSS2. Note that to make
this work, the path to the clone of `wxflow` needs to prepend that of
the virtual environment as the VE has a slightly older, fixed version of
wxflow.

Resolves #2124

Author: DavidHuber-NOAA

dev/ush/gw_setup.sh

@@ -29,10 +29,11 @@ fi
 
 # Set up the PYTHONPATH to include wxflow from HOMEgfs
 if [[ -d "${HOMEgfs}/sorc/wxflow/src" ]]; then
-  PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src"
+  PYTHONPATH="${HOMEgfs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}"
   export PYTHONPATH
 fi
 
+
 if [[ ${unset_homegfs} == "YES" ]]; then
   unset HOMEgfs
 fi

dev/ush/load_gw_run_modules.sh

@@ -82,7 +82,7 @@ fi
 
 # Set up the PYTHONPATH to include wxflow from HOMEgfs
 if [[ -d "${HOMEgfs}/sorc/wxflow/src" ]]; then
-  PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src"
+  PYTHONPATH="${HOMEgfs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}"
 fi
 
 # Add HOMEgfs/ush/python to PYTHONPATH

dev/ush/load_gw_verif_modules.sh

@@ -56,10 +56,7 @@ fi
 module use "${HOMEgfs}/modulefiles"
 
 case "${MACHINE_ID}" in
-"wcoss2")
-  target_module="gw_run.${MACHINE_ID}"
-  ;;
-"hera")
+"wcoss2" | "hera")
   target_module="gw_verif.${MACHINE_ID}"
   ;;
 *)

dev/ush/load_ufsda_modules.sh

@@ -80,7 +80,7 @@ pip list
 
 # Set up the PYTHONPATH to include wxflow from HOMEgfs
 if [[ -d "${HOMEgfs}/sorc/wxflow/src" ]]; then
-  PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src"
+  PYTHONPATH="${HOMEgfs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}"
 fi
 
 # Add HOMEgfs/ush/python to PYTHONPATH

dev/ush/load_ufswm_modules.sh

@@ -40,7 +40,7 @@ unset MACHINE_ID
 
 # Set up the PYTHONPATH to include wxflow from HOMEgfs
 if [[ -d "${HOMEgfs}/sorc/wxflow/src" ]]; then
-  PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src"
+  PYTHONPATH="${HOMEgfs}/sorc/wxflow/src${PYTHONPATH:+:${PYTHONPATH}}"
 fi
 
 # Add HOMEgfs/ush/python to PYTHONPATH

dev/workflow/rocoto/workflow_xml.py

@@ -2,7 +2,6 @@
 
 import os
 import stat
-from distutils.spawn import find_executable
 from datetime import datetime
 from collections import OrderedDict
 from typing import Dict
@@ -157,11 +156,13 @@ def _write_crontab(self, crontab_file: str = None, cronint: int = 5) -> None:
         """
 
         # No point creating a crontab if rocotorun is not available.
-        rocotoruncmd = find_executable('rocotorun')
-        if rocotoruncmd is None:
+        rocotorun = which('rocotorun')
+        if rocotorun is None:
             print('Failed to find rocotorun, crontab will not be created')
             return
 
+        rocotoruncmd = rocotorun.command
+
         rocotorunstr = f'{rocotoruncmd} -d {self.expdir}/{self.pslot}.db -w {self.expdir}/{self.pslot}.xml'
         cronintstr = f'*/{cronint} * * * *'
 

modulefiles/gw_run.wcoss2.lua

@@ -11,6 +11,9 @@ load(pathJoin("cfp", (os.getenv("cfp_ver") or "None")))
 setenv("USE_CFP","YES")
 
 load(pathJoin("python", (os.getenv("python_ver") or "None")))
+prepend_path("MODULEPATH", "/apps/dev/modulefiles")
+load(pathJoin("ve","gw", (os.getenv("gw_ve_ver") or "None")))
+
 load(pathJoin("gempak", (os.getenv("gempak_ver") or "None")))
 load(pathJoin("perl", (os.getenv("perl_ver") or "None")))
 load(pathJoin("libjpeg", (os.getenv("libjpeg_ver") or "None")))

modulefiles/gw_setup.wcoss2.lua

@@ -6,6 +6,9 @@ prepend_path("MODULEPATH", "/apps/ops/test/nco/modulefiles/core")
 load(pathJoin("rocoto","1.3.5"))
 load(pathJoin("PrgEnv-intel"))
 load(pathJoin("intel","19.1.3.304"))
-load(pathJoin("python", "3.8.6"))
+load(pathJoin("python", "3.12.0"))
+
+prepend_path("MODULEPATH", "/apps/dev/modulefiles")
+load(pathJoin("ve","gw","1.0"))
 
 whatis("Description: GFS run setup environment")

modulefiles/gw_verif.wcoss2.lua

@@ -0,0 +1,35 @@
+help([[
+Load environment to run verification on WCOSS2
+]])
+
+load(pathJoin("PrgEnv-intel", (os.getenv("PrgEnv_intel_ver") or "None")))
+load(pathJoin("craype", (os.getenv("craype_ver") or "None")))
+load(pathJoin("intel", (os.getenv("intel_ver") or "None")))
+load(pathJoin("cray-mpich", (os.getenv("cray_mpich_ver") or "None")))
+load(pathJoin("cray-pals", (os.getenv("cray_pals_ver") or "None")))
+load(pathJoin("cfp", (os.getenv("cfp_ver") or "None")))
+setenv("USE_CFP","YES")
+
+-- Use Python 3.8.6 for verification
+load(pathJoin("python", '3.8.6'))
+
+load(pathJoin("libjpeg", (os.getenv("libjpeg_ver") or "None")))
+
+-- MET was build with these versions of HDF5 and netCDF
+load(pathJoin("hdf5", '1.10.6'))
+load(pathJoin("netcdf", '4.7.4'))
+
+-- The cray library path for C MPI libraries (needed by C-only programs using netCDF)
+local cray_lib_path=os.getenv("CRAY_LD_LIBRARY_PATH") or ""
+prepend_path("LD_LIBRARY_PATH", cray_lib_path)
+
+load(pathJoin("prod_util", (os.getenv("prod_util_ver") or "None")))
+load(pathJoin("grib_util", (os.getenv("grib_util_ver") or "None")))
+load(pathJoin("wgrib2", (os.getenv("wgrib2_ver") or "None")))
+
+prepend_path("MODULEPATH", "/apps/ops/para/libs/modulefiles/compiler/intel/19.1.3.304")
+setenv("HPC_OPT", "/apps/ops/para/libs")
+load(pathJoin("met", (os.getenv("met_ver") or "None")))
+load(pathJoin("metplus", (os.getenv("metplus_ver") or "None")))
+
+whatis("Description: Verification environment")

versions/run.wcoss2.ver

@@ -9,7 +9,8 @@ export obsproc_ver=v1.1
 export imagemagick_ver=7.0.8-7
 export cdo_ver=1.9.8
 export perl_ver=5.32.0
-export python_ver=3.8.6
+export python_ver=3.12.0
+export gw_ve_ver=1.0
 export gempak_ver=7.15.1
 export nco_ver=5.0.6
 export grib_util_ver=1.2.3