From df89e12a8f035af29a83711b236a1b969a382db1 Mon Sep 17 00:00:00 2001 From: sanatcumar Date: Thu, 17 Aug 2023 13:22:26 +0000 Subject: [PATCH 01/13] Update ccpp-physics submodule. --- ccpp-physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp-physics b/ccpp-physics index 3a306a493..12c115e99 160000 --- a/ccpp-physics +++ b/ccpp-physics @@ -1 +1 @@ -Subproject commit 3a306a493a9a0b6c3c39c7b50d356f0ddb7c5c94 +Subproject commit 12c115e992d3a265eaaa67d72fcbdb3a6f21195f From d4cfcc0e49f87cb5c3d73db1e729c196da6411a9 Mon Sep 17 00:00:00 2001 From: sanatcumar Date: Thu, 17 Aug 2023 13:28:46 +0000 Subject: [PATCH 02/13] Update ccpp-physics --- ccpp-physics | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ccpp-physics b/ccpp-physics index 12c115e99..3a306a493 160000 --- a/ccpp-physics +++ b/ccpp-physics @@ -1 +1 @@ -Subproject commit 12c115e992d3a265eaaa67d72fcbdb3a6f21195f +Subproject commit 3a306a493a9a0b6c3c39c7b50d356f0ddb7c5c94 From 2b2c30351e8a954827f51ebee3392c4e0c4e16b0 Mon Sep 17 00:00:00 2001 From: sanatcumar Date: Mon, 3 Jun 2024 13:02:48 +0000 Subject: [PATCH 03/13] use_scale push --- sorc/sfc_climo_gen.fd/source_grid.F90 | 6 +++++- ush/sfc_climo_gen.sh | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/sorc/sfc_climo_gen.fd/source_grid.F90 b/sorc/sfc_climo_gen.fd/source_grid.F90 index 31f0b00e0..e6e309e5f 100644 --- a/sorc/sfc_climo_gen.fd/source_grid.F90 +++ b/sorc/sfc_climo_gen.fd/source_grid.F90 @@ -81,6 +81,7 @@ subroutine define_source_grid(localpet, npets, input_file) real(esmf_kind_r8), pointer :: lon_corner_ptr(:,:) real :: lon_extent real(esmf_kind_r4) :: missing + real(esmf_kind_r4) :: scale type(esmf_field) :: mask_field type(esmf_polekind_flag) :: polekindflag(2) @@ -214,9 +215,12 @@ subroutine define_source_grid(localpet, npets, input_file) call netcdf_err(status, "READING FIELD 1 ID") status=nf90_get_att(ncid, varid, 'missing_value', missing) call netcdf_err(status, "READING MISSING VALUE") - +! print *,"status=", status status = nf90_close(ncid) + + + !-------------------------------------------------------------------------- ! Create ESMF grid object for the source data grid. Check if ! data is periodic in the east/west direction. diff --git a/ush/sfc_climo_gen.sh b/ush/sfc_climo_gen.sh index ccab7f24a..834d81856 100755 --- a/ush/sfc_climo_gen.sh +++ b/ush/sfc_climo_gen.sh @@ -85,12 +85,14 @@ input_soil_color_file="${input_sfc_climo_dir}/soil_color.clm.0.05.nc" input_vegetation_type_file="${VEG_TYPE_FILE}" input_vegetation_greenness_file="${input_sfc_climo_dir}/vegetation_greenness.0.144.nc" mosaic_file_mdl="$mosaic_file" +input_leaf_area_index_file="/scratch2/NCEPDEV/land/Sanath.Kumar/ufs/lai/LAI_climo_pnnl.nc" orog_dir_mdl="$FIX_FV3" orog_files_mdl=$the_orog_files halo=$HALO maximum_snow_albedo_method="bilinear" snowfree_albedo_method="bilinear" vegetation_greenness_method="bilinear" +leaf_area_index_method="conserve" fract_vegsoil_type=${vegsoilt_frac} / EOF From 6e203b4325cd1da4445cbda91936ad2f6a46217b Mon Sep 17 00:00:00 2001 From: sanatcumar Date: Mon, 3 Jun 2024 13:58:30 +0000 Subject: [PATCH 04/13] changes to interep.F90 --- sorc/sfc_climo_gen.fd/interp.F90 | 63 +++++++++++++++++++++++++++++++- 1 file changed, 62 insertions(+), 1 deletion(-) diff --git a/sorc/sfc_climo_gen.fd/interp.F90 b/sorc/sfc_climo_gen.fd/interp.F90 index 877930724..1fb476f23 100644 --- a/sorc/sfc_climo_gen.fd/interp.F90 +++ b/sorc/sfc_climo_gen.fd/interp.F90 @@ -31,7 +31,7 @@ subroutine interp(localpet, method, input_file) integer :: varid, record integer :: tile_num, pt_loc_this_tile integer :: isrctermprocessing - + double precision :: scale integer(esmf_kind_i4), allocatable :: mask_mdl_one_tile(:,:) integer(esmf_kind_i4), pointer :: unmapped_ptr(:) @@ -110,7 +110,16 @@ subroutine interp(localpet, method, input_file) status = nf90_inq_varid(ncid, field_names(n), varid) call netcdf_err(status, "IN ROUTINE INTERP READING FIELD ID") status = nf90_get_var(ncid, varid, data_src_global, start=(/1,1,t/), count=(/i_src,j_src,1/)) + call netcdf_err(status, "IN ROUTINE INTERP READING FIELD") + status=nf90_get_att(ncid, varid, 'scale_factor', scale) + if (status /= 0) then + scale =1.0D0 + endif + print *,"scale=", scale + call scale_data(data_src_global,i_src,j_src,scale) + + endif print*,"- CALL FieldScatter FOR SOURCE GRID DATA." @@ -217,6 +226,7 @@ subroutine interp(localpet, method, input_file) print*,"- CALL FieldGather FOR MODEL GRID DATA." call ESMF_FieldGather(data_field_mdl, data_mdl_one_tile, rootPet=0, tile=tile, rc=rc) + !call scale_data (data_mdl_one_tile, i_mdl, j_mdl, scale) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__LINE__,file=__FILE__)) & call error_handler("IN FieldGather.", rc) @@ -241,6 +251,7 @@ subroutine interp(localpet, method, input_file) if (.not. fract_vegsoil_type) then select case (field_names(n)) case ('substrate_temperature','vegetation_greenness','leaf_area_index','slope_type','soil_type','soil_color') + !call scale_data (data_mdl_one_tile, i_mdl, j_mdl,scale) call adjust_for_landice (data_mdl_one_tile, vegt_mdl_one_tile, i_mdl, j_mdl, field_names(n)) end select endif @@ -257,6 +268,7 @@ subroutine interp(localpet, method, input_file) endif endif + enddo OUTPUT_LOOP if (allocated(vegt_mdl_one_tile)) deallocate(vegt_mdl_one_tile) @@ -306,6 +318,7 @@ subroutine adjust_for_landice(field, vegt, idim, jdim, field_ch) integer :: i, j, ierr real :: landice_value + select case (field_ch) case ('substrate_temperature') ! soil substrate temp @@ -372,3 +385,51 @@ subroutine adjust_for_landice(field, vegt, idim, jdim, field_ch) end select end subroutine adjust_for_landice + + + + +!> use Scale to fix the data to the correct value +!! +!! +!! +!! +!! @param[inout] idim i dimension of model tile. +!! @param[inout] jdim j dimension of model tile. +!! @param[in] field_ch Field name. +!! @author George Gayno NCEP/EMC +!! @author Sanath Kumar NCEP/EMC + subroutine scale_data(field,idim,jdim,scale) + + use esmf + use mpi + + implicit none + + + + integer, intent(in) :: idim, jdim + + + real(esmf_kind_r4), intent(inout) :: field(idim,jdim) + + + integer :: i, j, ierr + double precision :: scale + + + + + do j = 1, jdim + do i = 1, idim + field(i,j) = field(i,j)*scale + + enddo + enddo + + end subroutine scale_data + + + + + From f16e80df77bc72729461465a7d8ad84367e6b961 Mon Sep 17 00:00:00 2001 From: sanatcumar Date: Mon, 3 Jun 2024 14:11:35 +0000 Subject: [PATCH 05/13] removed unnecessary declerations --- sorc/sfc_climo_gen.fd/source_grid.F90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/sorc/sfc_climo_gen.fd/source_grid.F90 b/sorc/sfc_climo_gen.fd/source_grid.F90 index e6e309e5f..23b63725b 100644 --- a/sorc/sfc_climo_gen.fd/source_grid.F90 +++ b/sorc/sfc_climo_gen.fd/source_grid.F90 @@ -81,7 +81,6 @@ subroutine define_source_grid(localpet, npets, input_file) real(esmf_kind_r8), pointer :: lon_corner_ptr(:,:) real :: lon_extent real(esmf_kind_r4) :: missing - real(esmf_kind_r4) :: scale type(esmf_field) :: mask_field type(esmf_polekind_flag) :: polekindflag(2) @@ -215,7 +214,6 @@ subroutine define_source_grid(localpet, npets, input_file) call netcdf_err(status, "READING FIELD 1 ID") status=nf90_get_att(ncid, varid, 'missing_value', missing) call netcdf_err(status, "READING MISSING VALUE") -! print *,"status=", status status = nf90_close(ncid) From a44ee4df8b0d2c1665f102c2d7d6f1b1a8ad92a8 Mon Sep 17 00:00:00 2001 From: sanatcumar Date: Mon, 3 Jun 2024 19:06:25 +0000 Subject: [PATCH 06/13] Fixed unnecessary logic --- sorc/sfc_climo_gen.fd/interp.F90 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sorc/sfc_climo_gen.fd/interp.F90 b/sorc/sfc_climo_gen.fd/interp.F90 index 1fb476f23..79c88c619 100644 --- a/sorc/sfc_climo_gen.fd/interp.F90 +++ b/sorc/sfc_climo_gen.fd/interp.F90 @@ -113,11 +113,11 @@ subroutine interp(localpet, method, input_file) call netcdf_err(status, "IN ROUTINE INTERP READING FIELD") status=nf90_get_att(ncid, varid, 'scale_factor', scale) - if (status /= 0) then - scale =1.0D0 + if (status == 0) then + call scale_data(data_src_global,i_src,j_src,scale) endif - print *,"scale=", scale - call scale_data(data_src_global,i_src,j_src,scale) +! print *,"scale=", scale +! call scale_data(data_src_global,i_src,j_src,scale) endif From 1117f4f982eaa657338ef20801a98c6ffcee066c Mon Sep 17 00:00:00 2001 From: sanatcumar Date: Mon, 3 Jun 2024 19:08:41 +0000 Subject: [PATCH 07/13] Removed commented statements --- sorc/sfc_climo_gen.fd/interp.F90 | 2 -- 1 file changed, 2 deletions(-) diff --git a/sorc/sfc_climo_gen.fd/interp.F90 b/sorc/sfc_climo_gen.fd/interp.F90 index 79c88c619..30b2de47b 100644 --- a/sorc/sfc_climo_gen.fd/interp.F90 +++ b/sorc/sfc_climo_gen.fd/interp.F90 @@ -226,7 +226,6 @@ subroutine interp(localpet, method, input_file) print*,"- CALL FieldGather FOR MODEL GRID DATA." call ESMF_FieldGather(data_field_mdl, data_mdl_one_tile, rootPet=0, tile=tile, rc=rc) - !call scale_data (data_mdl_one_tile, i_mdl, j_mdl, scale) if(ESMF_logFoundError(rcToCheck=rc,msg=ESMF_LOGERR_PASSTHRU,line=__LINE__,file=__FILE__)) & call error_handler("IN FieldGather.", rc) @@ -251,7 +250,6 @@ subroutine interp(localpet, method, input_file) if (.not. fract_vegsoil_type) then select case (field_names(n)) case ('substrate_temperature','vegetation_greenness','leaf_area_index','slope_type','soil_type','soil_color') - !call scale_data (data_mdl_one_tile, i_mdl, j_mdl,scale) call adjust_for_landice (data_mdl_one_tile, vegt_mdl_one_tile, i_mdl, j_mdl, field_names(n)) end select endif From b6a2460511f2e1ca1c36be8dd40c1c656b7260c5 Mon Sep 17 00:00:00 2001 From: sanatcumar Date: Mon, 3 Jun 2024 20:45:31 +0000 Subject: [PATCH 08/13] Removed prints/ commented statements and empty spaces --- sorc/sfc_climo_gen.fd/interp.F90 | 28 ++++++---------------------- 1 file changed, 6 insertions(+), 22 deletions(-) diff --git a/sorc/sfc_climo_gen.fd/interp.F90 b/sorc/sfc_climo_gen.fd/interp.F90 index 30b2de47b..752b7f778 100644 --- a/sorc/sfc_climo_gen.fd/interp.F90 +++ b/sorc/sfc_climo_gen.fd/interp.F90 @@ -115,10 +115,7 @@ subroutine interp(localpet, method, input_file) status=nf90_get_att(ncid, varid, 'scale_factor', scale) if (status == 0) then call scale_data(data_src_global,i_src,j_src,scale) - endif -! print *,"scale=", scale -! call scale_data(data_src_global,i_src,j_src,scale) - + endif endif @@ -389,42 +386,29 @@ end subroutine adjust_for_landice !> use Scale to fix the data to the correct value !! -!! -!! -!! !! @param[inout] idim i dimension of model tile. !! @param[inout] jdim j dimension of model tile. !! @param[in] field_ch Field name. !! @author George Gayno NCEP/EMC !! @author Sanath Kumar NCEP/EMC - subroutine scale_data(field,idim,jdim,scale) +!! +subroutine scale_data(field,idim,jdim,scale) use esmf use mpi implicit none - - - integer, intent(in) :: idim, jdim - - + integer :: i, j real(esmf_kind_r4), intent(inout) :: field(idim,jdim) - - - integer :: i, j, ierr double precision :: scale - - - do j = 1, jdim - do i = 1, idim + do i = 1, idim field(i,j) = field(i,j)*scale + enddo enddo - enddo - end subroutine scale_data From e2b4d0a704f2f03abd3e1c04f2284ba7021f0f15 Mon Sep 17 00:00:00 2001 From: sanatcumar Date: Tue, 25 Jun 2024 21:01:09 +0000 Subject: [PATCH 09/13] Added doxygen info --- sorc/sfc_climo_gen.fd/interp.F90 | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/sorc/sfc_climo_gen.fd/interp.F90 b/sorc/sfc_climo_gen.fd/interp.F90 index 752b7f778..aa8473670 100644 --- a/sorc/sfc_climo_gen.fd/interp.F90 +++ b/sorc/sfc_climo_gen.fd/interp.F90 @@ -385,10 +385,11 @@ end subroutine adjust_for_landice !> use Scale to fix the data to the correct value -!! -!! @param[inout] idim i dimension of model tile. -!! @param[inout] jdim j dimension of model tile. -!! @param[in] field_ch Field name. +!! +!! @param[inout] field to scale. +!! @param[in] idim i dimension of model tile. +!! @param[in] jdim j dimension of model tile. +!! @param[inout] scale factor to scale the field. !! @author George Gayno NCEP/EMC !! @author Sanath Kumar NCEP/EMC !! From 77897315133213aa58fb4b2fc8af6beeaa992b8c Mon Sep 17 00:00:00 2001 From: sanatcumar Date: Tue, 25 Jun 2024 21:02:27 +0000 Subject: [PATCH 10/13] Doxygen udpates --- sorc/sfc_climo_gen.fd/interp.F90 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sorc/sfc_climo_gen.fd/interp.F90 b/sorc/sfc_climo_gen.fd/interp.F90 index aa8473670..723ba26d6 100644 --- a/sorc/sfc_climo_gen.fd/interp.F90 +++ b/sorc/sfc_climo_gen.fd/interp.F90 @@ -389,7 +389,7 @@ end subroutine adjust_for_landice !! @param[inout] field to scale. !! @param[in] idim i dimension of model tile. !! @param[in] jdim j dimension of model tile. -!! @param[inout] scale factor to scale the field. +!! @param[in] scale factor to scale the field. !! @author George Gayno NCEP/EMC !! @author Sanath Kumar NCEP/EMC !! From 6de101083241babfe623fa8a42c96fd8aa609cd5 Mon Sep 17 00:00:00 2001 From: sanatcumar Date: Wed, 21 Aug 2024 12:17:01 +0000 Subject: [PATCH 11/13] updated the fix links --- ush/sfc_climo_gen.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ush/sfc_climo_gen.sh b/ush/sfc_climo_gen.sh index 834d81856..6cf740f2f 100755 --- a/ush/sfc_climo_gen.sh +++ b/ush/sfc_climo_gen.sh @@ -85,7 +85,7 @@ input_soil_color_file="${input_sfc_climo_dir}/soil_color.clm.0.05.nc" input_vegetation_type_file="${VEG_TYPE_FILE}" input_vegetation_greenness_file="${input_sfc_climo_dir}/vegetation_greenness.0.144.nc" mosaic_file_mdl="$mosaic_file" -input_leaf_area_index_file="/scratch2/NCEPDEV/land/Sanath.Kumar/ufs/lai/LAI_climo_pnnl.nc" +input_leaf_area_index_file="${input_sfc_climo_dir}/LAI_climo_pnnl.nc" orog_dir_mdl="$FIX_FV3" orog_files_mdl=$the_orog_files halo=$HALO From 1401c521261244a1add705989c888a1d546c52bc Mon Sep 17 00:00:00 2001 From: Sanath Kumar Date: Tue, 5 Nov 2024 12:28:54 -0600 Subject: [PATCH 12/13] addressing Orion memory out issues --- driver_scripts/driver_grid.orion.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/driver_scripts/driver_grid.orion.sh b/driver_scripts/driver_grid.orion.sh index 656afcf33..63018f873 100755 --- a/driver_scripts/driver_grid.orion.sh +++ b/driver_scripts/driver_grid.orion.sh @@ -6,8 +6,9 @@ #SBATCH -o log.fv3_grid_driver #SBATCH -e log.fv3_grid_driver #SBATCH --nodes=2 --ntasks-per-node=15 -#SBATCH -q debug -#SBATCH -t 00:30:00 +#SBATCH --partition=bigmem +#SBATCH -q batch +#SBATCH -t 04:30:00 #----------------------------------------------------------------------- # Driver script to create a cubic-sphere based model grid on Orion. From c62fab082bc1726b286757bc9de759d1874a1c76 Mon Sep 17 00:00:00 2001 From: Sanath Kumar Date: Tue, 5 Nov 2024 14:47:32 -0600 Subject: [PATCH 13/13] Orion Memory issues --- driver_scripts/driver_grid.orion.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/driver_scripts/driver_grid.orion.sh b/driver_scripts/driver_grid.orion.sh index 63018f873..b1b35d23b 100755 --- a/driver_scripts/driver_grid.orion.sh +++ b/driver_scripts/driver_grid.orion.sh @@ -5,7 +5,7 @@ #SBATCH --open-mode=truncate #SBATCH -o log.fv3_grid_driver #SBATCH -e log.fv3_grid_driver -#SBATCH --nodes=2 --ntasks-per-node=15 +#SBATCH --nodes=6 --ntasks-per-node=12 #SBATCH --partition=bigmem #SBATCH -q batch #SBATCH -t 04:30:00