Merge branch 'develop' into feature/map_chans

.github/workflows/ci.yml

@@ -7,6 +7,10 @@ on:
   pull_request:
     branches: [develop]
 
+concurrency:
+  group: ${{ github.ref }}
+  cancel-in-progress: ${{ github.ref != 'refs/heads/develop' }}
+
 env:
   REPO: ${{ github.event.repository.name }}
   BUNDLE: pr-${{ github.event.number }}
@@ -16,116 +20,118 @@ jobs:
     name: gnu 11.3.1 (UKMO)
     if: github.repository_owner == 'MetOffice'
     runs-on: jedi-self-hosted-rg
-    
+
     steps:
-      - name: pre-submit cleanup
+      - name: Pre-submit cleanup
         run: |
-          ls -la ./
           rm -fr ./${{ env.BUNDLE }} || true
-          ls -la ./
-      - name: checkout current
+          ls -lgo ./
+      - name: Checkout current
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/${{ env.REPO }}
-      - name: initiate bundle
+      - name: Initiate bundle
         run: cp ./${{ env.BUNDLE }}/${{ env.REPO }}/ci/* ./${{ env.BUNDLE }}/
 
-      - name: checkout jedicmake
+      - name: Checkout jedicmake
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/jedicmake
           repository: JCSDA-internal/jedi-cmake
           submodules: true
           token: ${{ secrets.GH_PAT }}
 
-      - name: checkout oops
+      - name: Checkout oops
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/oops
           repository: JCSDA-internal/oops
           token: ${{ secrets.GH_PAT }}
-      - name: matching branch checkout attempt oops ${{ github.head_ref }}
+      - name: Checkout attempt for oops@${{ github.head_ref }}
         if: github.event_name == 'pull_request'
         run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops
 
-      - name: checkout ioda
+      - name: Checkout ioda
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/ioda
           repository: JCSDA-internal/ioda
           token: ${{ secrets.GH_PAT }}
-      - name: matching branch checkout attempt ioda ${{ github.head_ref }}
+      - name: Checkout attempt for ioda@${{ github.head_ref }}
         if: github.event_name == 'pull_request'
         run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ioda
 
-      - name: checkout ufo
+      - name: Checkout ufo
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/ufo
           repository: JCSDA-internal/ufo
           token: ${{ secrets.GH_PAT }}
-      - name: matching branch checkout attempt ufo ${{ github.head_ref }}
+      - name: Checkout attempt for ufo@${{ github.head_ref }}
         if: github.event_name == 'pull_request'
         run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ufo
 
-      - name: build and test
+      - name: Build and Test
         run: |
           az acr login --name ngmssboxjediacr
           docker run --rm \
             --entrypoint=/usr/local/src/${REPO}/${BUNDLE}/build-and-test \
             --workdir=/usr/local/src/${REPO}/${BUNDLE} \
             --volume ${PWD}/${BUNDLE}:/usr/local/src/${REPO}/${BUNDLE} \
-            'ngmssboxjediacr.azurecr.io/jedibase:alma9' \
-          && rm -fr ${{github.workspace}}/${{env.BUNDLE}}
+            'ngmssboxjediacr.azurecr.io/jedibase:alma9'
+
+      - name: Cleanup workspace
+        working-directory: ${{github.workspace}}
+        run: rm -fr ${{ env.BUNDLE }} || true
 
   build2:
     name: gnu 9.4.0 (JCSDA)
     if: github.repository_owner == 'MetOffice'
     runs-on: ubuntu-latest
 
     steps:
-      - name: checkout current
+      - name: Checkout current
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/${{ env.REPO }}
       - name: initiate bundle
         run: cp ./${{ env.BUNDLE }}/${{ env.REPO }}/ci/* ./${{ env.BUNDLE }}/
 
-      - name: checkout jedicmake
+      - name: Checkout jedicmake
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/jedicmake
           repository: JCSDA-internal/jedi-cmake
           submodules: true
           token: ${{ secrets.GH_PAT }}
 
-      - name: checkout oops
+      - name: Checkout oops
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/oops
           repository: JCSDA-internal/oops
           token: ${{ secrets.GH_PAT }}
-      - name: matching branch checkout attempt oops ${{ github.head_ref }}
+      - name: Checkout attempt for oops@${{ github.head_ref }}
         if: github.event_name == 'pull_request'
         run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops
 
-      - name: checkout ioda
+      - name: Checkout ioda
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/ioda
           repository: JCSDA-internal/ioda
           token: ${{ secrets.GH_PAT }}
-      - name: matching branch checkout attempt ioda ${{ github.head_ref }}
+      - name: Checkout attempt for ioda@${{ github.head_ref }}
         if: github.event_name == 'pull_request'
         run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ioda
 
-      - name: checkout ufo
+      - name: Checkout ufo
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/ufo
           repository: JCSDA-internal/ufo
           token: ${{ secrets.GH_PAT }}
-      - name: matching branch checkout attempt ufo ${{ github.head_ref }}
+      - name: Checkout attempt for ufo@${{ github.head_ref }}
         if: github.event_name == 'pull_request'
         run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ufo
 
@@ -143,52 +149,52 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - name: checkout current
+      - name: Checkout current
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/${{ env.REPO }}
       - name: initiate bundle
         run: cp ./${{ env.BUNDLE }}/${{ env.REPO }}/ci/* ./${{ env.BUNDLE }}/
 
-      - name: checkout jedicmake
+      - name: Checkout jedicmake
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/jedicmake
           repository: JCSDA-internal/jedi-cmake
           submodules: true
           token: ${{ secrets.GH_PAT }}
 
-      - name: checkout oops
+      - name: Checkout oops
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/oops
           repository: JCSDA-internal/oops
           token: ${{ secrets.GH_PAT }}
-      - name: matching branch checkout attempt oops ${{ github.head_ref }}
+      - name: Checkout attempt for oops@${{ github.head_ref }}
         if: github.event_name == 'pull_request'
         run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/oops
 
-      - name: checkout ioda
+      - name: Checkout ioda
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/ioda
           repository: JCSDA-internal/ioda
           token: ${{ secrets.GH_PAT }}
-      - name: matching branch checkout attempt ioda ${{ github.head_ref }}
+      - name: Checkout attempt for ioda@${{ github.head_ref }}
         if: github.event_name == 'pull_request'
         run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ioda
 
-      - name: checkout ufo
+      - name: Checkout ufo
         uses: actions/checkout@v4
         with:
           path: ./${{ env.BUNDLE }}/ufo
           repository: JCSDA-internal/ufo
           token: ${{ secrets.GH_PAT }}
-      - name: matching branch checkout attempt ufo ${{ github.head_ref }}
+      - name: Checkout attempt for ufo@${{ github.head_ref }}
         if: github.event_name == 'pull_request'
         run: ./${{ env.BUNDLE }}/get-ref-branch ./${{ env.BUNDLE }}/ufo
 
-      - name: build and test
+      - name: Build and Test
         run: |
           docker run --rm \
             --entrypoint=/usr/local/src/${REPO}/${BUNDLE}/build-and-test \

ci/CMakeLists.txt

@@ -7,6 +7,8 @@ project( opsinputs-ci VERSION 0.1 LANGUAGES C CXX Fortran )
 
 set( ENABLE_MPI ON CACHE BOOL "Compile with MPI" )
 set( ENABLE_OMP ON CACHE BOOL "Compile with OpenMP" )
+option(ENABLE_IODA_DATA "Obtain ioda test data from ioda-data repository (vs tarball)" ON)
+option(ENABLE_UFO_DATA "Obtain ufo test data from ufo-data repository (vs tarball)" ON)
 
 add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/jedicmake")
 if(NOT DEFINED jedicmake_DIR)

ci/build-and-test

@@ -2,6 +2,8 @@
 #
 # (C) Crown Copyright 2023, the Met Office. All rights reserved.
 #
+# shellcheck disable=SC2317
+# shellcheck source=/dev/null
 set -euo pipefail
 
 finally() {
@@ -11,6 +13,10 @@ finally() {
         cd /
         rm -fr "${WORKD}"
     fi
+    if [[ -d ${BASE:-} ]]; then
+        cd /
+        rm -fr "$BASE" 2>/dev/null || true
+    fi
 }
 
 # -- HERE is /usr/local/src/<REPONAME>/pr-<#> (cf ../.github/workflow/ci.yml)

deps/ops/CMakeLists.txt

@@ -28,8 +28,6 @@ set(OPS_SOURCE_FILES
   code/OpsMod_ModelColumnIO/OpsMod_ModelColumnIO.f90
   code/OpsMod_ModelIO/OpsMod_ModelIO.f90
   code/OpsMod_ModelObInfo/OpsMod_ModelObInfo.f90
-  code/OpsMod_ODB/OpsMod_ODBKinds.f90
-  code/OpsMod_ODB/OpsMod_ODBTypes.f90
   code/OpsMod_ObTypeUtils/OpsMod_ObTypeUtils.f90
   code/OpsMod_ObsInfo/OpsMod_MiscTypes.f90
   code/OpsMod_ObsInfo/OpsMod_ObsInfo.f90
@@ -41,7 +39,6 @@ set(OPS_SOURCE_FILES
   code/OpsMod_UMImport/ops_portio2a.c
   code/OpsMod_UMImport/ops_portutils.c
   code/OpsMod_Utilities/OpsMod_Utilities.f90
-  code/OpsMod_Varobs/OpsMod_Varobs.f90
   code/OpsMod_VarobsLib/OpsMod_VarobsLib.f90
   code/OpsProg_Utils/OpsMod_UtilsProgs.f90
   public/GenMod_Control/GenMod_Control.F90
@@ -66,27 +63,30 @@ set(OPS_SOURCE_FILES
   public/OpsMod_Sort/OpsMod_Sort.f90
   public/OpsMod_UMInterface/OpsMod_UMInterface.f90
   public/OpsMod_VerticalInterp/OpsMod_VerticalInterp.f90
-  public/Ops_Constants/GenMod_CLookAdd.f90
-  public/Ops_Constants/GenMod_Constants.f90
-  public/Ops_Constants/GenMod_MiscUMScienceConstants.f90
-  public/Ops_Constants/GenMod_UMHeaderConstants.f90
-  public/Ops_Constants/OpsMod_Ancil.f90
-  public/Ops_Constants/OpsMod_CXIndexes.f90
-  public/Ops_Constants/OpsMod_Constants.f90
-  public/Ops_Constants/OpsMod_Kinds.F90
-  public/Ops_Constants/OpsMod_ObsGroupInfo.f90
-  public/Ops_Constants/OpsMod_ObsTypes.f90
-  public/Ops_Constants/OpsMod_SatIds.f90
-  public/Ops_Constants/OpsMod_Stash.f90
-  public/Ops_Constants/OpsMod_Varfields.f90
+  stubs/Ops_Constants/GenMod_CLookAdd.f90
+  stubs/Ops_Constants/GenMod_Constants.f90
+  stubs/Ops_Constants/GenMod_MiscUMScienceConstants.f90
+  stubs/Ops_Constants/GenMod_UMHeaderConstants.f90
+  stubs/Ops_Constants/OpsMod_Ancil.f90
+  stubs/Ops_Constants/OpsMod_CXIndexes.f90
+  stubs/Ops_Constants/OpsMod_Constants.f90
+  stubs/Ops_Constants/OpsMod_Kinds.F90
+  stubs/Ops_Constants/OpsMod_ObsGroupInfo.f90
+  stubs/Ops_Constants/OpsMod_ObsTypes.f90
+  stubs/Ops_Constants/OpsMod_SatIds.f90
+  stubs/Ops_Constants/OpsMod_Stash.f90
+  stubs/Ops_Constants/OpsMod_Varfields.f90
   stubs/OpsMod_AODGeneral/OpsMod_AODGeneral.f90
   stubs/OpsMod_GPSRO/OpsMod_GPSRO.f90
   stubs/OpsMod_GPSRO/OpsMod_GPSROInfo.f90
   stubs/OpsMod_GPSRO/OpsMod_GPSROInfo_BA.f90
   stubs/OpsMod_GroundGPS/OpsMod_GroundGPS.f90
+  stubs/OpsMod_ODB/OpsMod_ODBKinds.f90
+  stubs/OpsMod_ODB/OpsMod_ODBTypes.f90
   stubs/OpsMod_Radar/OpsMod_Radar.f90
   stubs/OpsMod_RadarZ/OpsMod_RadarZ.f90
   stubs/OpsMod_Sonde/OpsMod_SondeDrift.f90
+  stubs/OpsMod_Varobs/OpsMod_Varobs.f90
   stubs/Ops_SatRad_Info/OpsMod_SatRad_RTmodel.f90
   stubs/Ops_SatRad_SetUp/OpsMod_SatRad_SetUp.f90
 )

deps/ops/code/OpsMod_CXGenerate/Ops_GetDefaultCxFields.inc → deps/ops/stubs/OpsMod_CXGenerate/Ops_GetDefaultCxFields.inc

@@ -235,7 +235,7 @@ SELECT CASE (ObsGroup)
   CASE (ObsGroupSBUV)
     CxFields(1:7) = (/StashItem_ozone_old,StashItem_p,StashCode_t2, &
                       StashCode_rh2,StashCode_t_p,StashCode_rh_p,StashCode_pmsl/)
-  CASE (ObsGroupScatwind)
+  CASE (ObsGroupScatwind, ObsGroupScatwindChosen)
     CxFields(1:12) = (/StashItem_u,StashItem_v,StashItem_theta, StashItem_SST, &
                        StashItem_SeaIce,StashItem_orog,StashItem_p,            &
                        StashItem_p_Surface,StashCode_u10,StashCode_v10,        &

deps/ops/code/OpsMod_ODB/OpsFn_ObsToReportType.inc → deps/ops/stubs/OpsMod_ODB/OpsFn_ObsToReportType.inc

@@ -931,7 +931,7 @@ SELECT CASE (Obs % Header % ObsGroup)
         END IF
       END DO
     END IF
-  CASE (ObsGroupScatwind)
+  CASE (ObsGroupScatwind, ObsGroupScatwindChosen)
     IF (Obs % Header % Satid % Present) THEN
       DO i = ob_start, ob_end
         IF (Obs % Satid(i) == SatId_MetopB) THEN
@@ -963,7 +963,8 @@ SELECT CASE (Obs % Header % ObsGroup)
         ELSE
           found_report_type = .FALSE.
           DO j = 1, SIZE (map, DIM = 1)
-            IF (map(j,1) == ObsGroupScatwind .AND. map(j,2) == Obs % Satid(i)) THEN
+            IF ((map(j,1) == ObsGroupScatwind .AND. map(j,2) == Obs % Satid(i)) .OR. &
+                (map(j,1) == ObsGroupScatwindChosen .AND. map(j,2) == Obs % Satid(i))) THEN
               report_types(i - ob_start + 1) = map(j,3)
               found_report_type = .TRUE.
               EXIT

deps/ops/code/OpsMod_Varobs/Ops_CheckVarFields.inc → deps/ops/stubs/OpsMod_Varobs/Ops_CheckVarFields.inc

@@ -38,7 +38,8 @@ USE OpsMod_MiscTypes, ONLY: &
 
 USE OpsMod_ObsGroupInfo, ONLY: &
     ObsGroupSurface,           &
-    ObsGroupScatwind
+    ObsGroupScatwind,          &
+    ObsGroupScatwindChosen
 
 USE OpsMod_ObsInfo, ONLY: &
     OB_type
@@ -113,14 +114,16 @@ DO Ivar = 1, ActualMaxVarfield
       END IF
     CASE (VarField_u)
       IF (Ob % header % ObsGroup == ObsGroupSurface .OR. &
-          Ob % header % ObsGroup == ObsGroupScatwind) THEN
+          Ob % header % ObsGroup == ObsGroupScatwind .OR. &
+          Ob % header % ObsGroup == ObsGroupScatwindChosen) THEN
         ObHdrVrbl = Ob % header % u10
       ELSE
         ObHdrVrbl = Ob % header % u
       END IF
     CASE (VarField_v)
       IF (Ob % header % ObsGroup == ObsGroupSurface .OR. &
-          Ob % header % ObsGroup == ObsGroupScatwind) THEN
+          Ob % header % ObsGroup == ObsGroupScatwind .OR. &
+          Ob % header % ObsGroup == ObsGroupScatwindChosen) THEN
         ObHdrVrbl = Ob % header % v10
       ELSE
         ObHdrVrbl = Ob % header % v

deps/ops/code/OpsMod_Varobs/Ops_GetDefaultVarfields.inc → deps/ops/stubs/OpsMod_Varobs/Ops_GetDefaultVarfields.inc

@@ -240,6 +240,8 @@ SELECT CASE (ObsGroup)
     Varfields(1:1) = (/VarField_SBUVozone/)
   CASE (ObsGroupScatwind)
     Varfields(1:3) = (/Varfield_u10ambwind,varfield_v10ambwind,Varfield_pcorrect/)
+  CASE (ObsGroupScatwindChosen)
+    Varfields(1:2) = (/Varfield_u,varfield_v/)
   CASE (ObsGroupSEVIRIClr)
     Varfields(1:12) = (/Varfield_britemp,Varfield_satid,Varfield_satzenith, &
                         Varfield_solzenith,Varfield_tskin,Varfield_tcozone, &

deps/ops/code/OpsMod_Varobs/Ops_SetupVarArray.inc → deps/ops/stubs/OpsMod_Varobs/Ops_SetupVarArray.inc

@@ -56,6 +56,7 @@ USE OpsMod_ObsGroupInfo, ONLY: &
     ObsGroupGPSRO,             &
     ObsGroupSatwind,           &
     ObsGroupScatwind,          &
+    ObsGroupScatwindChosen,    &
     ObsGroupSurface
 
 USE OpsMod_ObsInfo, ONLY: &
@@ -253,7 +254,8 @@ DO Ivar = 1, NumValidVarFields
       END IF
     CASE (VarField_u)
       IF (Ob % header % ObsGroup == ObsGroupSurface .OR. &
-          Ob % header % ObsGroup == ObsGroupScatwind) THEN
+          Ob % header % ObsGroup == ObsGroupScatwind .OR. &
+          Ob % header % ObsGroup == ObsGroupScatwindChosen) THEN
         ObHdrVrbl = Ob % header % u10
         ObVrblSingle => Ob % u10(:)
         Pge_1D => Pge % u10
@@ -264,7 +266,8 @@ DO Ivar = 1, NumValidVarFields
       END IF
     CASE (VarField_v)
       IF (Ob % header % ObsGroup == ObsGroupSurface .OR. &
-          Ob % header % ObsGroup == ObsGroupScatwind) THEN
+          Ob % header % ObsGroup == ObsGroupScatwind .OR. &
+          Ob % header % ObsGroup == ObsGroupScatwindChosen) THEN
         ObHdrVrbl = Ob % header % v10
         ObVrblSingle => Ob % v10(:)
         Pge_1D => Pge % u10