Skip to content

Commit 3b90625

Browse files
gutmanngutmann
authored
Version 2.1 release
Version 2.1 official release, more documentation to follow.
2 parents a442cc7 + 0a53597 commit 3b90625

82 files changed

Lines changed: 69968 additions & 2204 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎.github/scripts/icar_install_utils.sh‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -74,21 +74,24 @@ function icar_dependencies {
7474
sudo apt-get update
7575
sudo apt-get install libcurl4-gnutls-dev
7676
sudo apt-get install libfftw3-dev
77+
sudo apt-get install netcdf-bin
78+
sudo apt-get install libnetcdff-dev
79+
7780
# Installing HDF5 currently not working for NetCDF
7881
# sudo apt-get install libhdf5-dev libhdf5-openmpi-dev
7982

8083
export CPPFLAGS="$CPPFLAGS -I${INSTALLDIR}/include"
8184
export LDFLAGS="$LDFLAGS -L${INSTALLDIR}/lib"
8285

83-
# Install szip (used by hdf5)
84-
install_szip
85-
# Install HDF5
86-
install_hdf5
86+
# # Install szip (used by hdf5)
87+
# install_szip
88+
# # Install HDF5
89+
# install_hdf5
8790

88-
# Install NetCDF-C
89-
install_netcdf_c
90-
# Install NetCDF fortran
91-
install_netcdf_fortran
91+
# # Install NetCDF-C
92+
# install_netcdf_c
93+
# # Install NetCDF fortran
94+
# install_netcdf_fortran
9295

9396
# put installed bin directory in PATH
9497
export PATH=${INSTALLDIR}/bin:$PATH
@@ -148,7 +151,17 @@ function gen_test_run_data {
148151
function execute_test_run {
149152
cp ${GITHUB_WORKSPACE}/src/icar ${GITHUB_WORKSPACE}/tests/
150153
cd ${GITHUB_WORKSPACE}/tests
151-
./icar icar_options.nm
154+
echo "Starting ICAR run"
155+
./icar icar_options.nml
156+
time_dim=$(ncdump -v time icar_out_000001_2020-12-01_00-00-00.nc | grep "time = UNLIMITED" | sed 's/[^0-9]*//g')
157+
158+
if [[ ${time_dim} == "1" ]]; then
159+
echo "FAILURE: ICAR output time dimension should not be equal to one, it was ${time_dim}"
160+
exit 1
161+
else
162+
echo "SUCCESS: time dimension is equal to ${time_dim}"
163+
exit 0
164+
fi
152165
}
153166

154167
function icar_after_success {

‎helpers/aggregate_parallel_files.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ def set_up_dataset(d):
7676
data = np.zeros((nt, nz, ny + y_off, nx + x_off))
7777

7878
# print(name, data.shape, dims, attrs)
79-
data_vars[v] = xr.DataArray(data.astype(np.float32), dims=dims, name=name, attrs=attrs)#, coords=coords)
79+
data_vars[v] = xr.DataArray(data.astype(d[v].dtype), dims=dims, name=name, attrs=attrs)#, coords=coords)
8080

8181
ds = xr.Dataset(data_vars, attrs=d.attrs)
8282
ds.encoding = d.encoding

helpers/batch_submit.sh renamed to helpers/batch_submit_LSF.sh

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
#
55
#BSUB -P P48500028 # project code
66
#BSUB -W 12:00 # wall-clock time (hrs:mins)
7-
#BSUB -n 1 # number of tasks in job
7+
#BSUB -n 1 # number of tasks in job
88
#BSUB -R "span[ptile=16]" # run 16 MPI tasks per node
99
#BSUB -J run_name # job name
1010
#BSUB -o job_output/run_name.%J.out # job output file (%J is replaced by the job ID)
@@ -24,7 +24,7 @@ PREFIX=run_name
2424
# it is useful to keep all other filenames relative to $PREFIX
2525
# note that this is not required anywhere though
2626
OUTDIR=$PREFIX
27-
OPTFILE=${PREFIX}_options.nml
27+
OPTFILE=${PREFIX}_options.nml
2828
BATCHFILE=${PREFIX}_batch_submit.sh
2929
TEMPLATE=${PREFIX}_template.nml
3030

@@ -58,17 +58,17 @@ if [[ ! -e ${PREFIX}_finished ]]; then
5858
if [[ -e ${PREFIX}_running ]]; then
5959
$SETUP_RUN $OPTFILE $TEMPLATE
6060
fi
61-
61+
6262
# declare that we have run before so the next job will know
6363
touch ${PREFIX}_running
64-
64+
6565
# run the actual executable (e.g. icar options.nml)
6666
$EXE $OPTFILE
6767
# typically the job will get killed while icar is running
68-
# but for some reason bkilling the job still lets it touch _finished...
68+
# but for some reason bkilling the job still lets it touch _finished...
6969
# maybe this will give it a chance to really kill it first?
7070
sleep 10
71-
71+
7272
# if icar completes, we are done, tell the next job that we finished
7373
touch ${PREFIX}_finished
7474
else

‎helpers/batch_submit_PBS.sh‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#!/bin/bash
2+
#
3+
4+
### Job Name (will be used as prefix later on!)
5+
#PBS -N icar_batch_run
6+
### Project code
7+
#PBS -A P48500028
8+
#PBS -l walltime=00:15:00
9+
#PBS -q regular
10+
### Merge output and error files
11+
#PBS -o job_output/log.out
12+
### job error file (PBS will not allow use of ${PBS_JOBID} here? )
13+
#PBS -e job_output/log.err
14+
### Select X nodes with 36 CPUs each for a total of 72 MPI processes
15+
#PBS -l select=1:ncpus=36:mpiprocs=36:ompthreads=1
16+
17+
### PBS options for automation: https://gif.biotech.iastate.edu/submitting-dependency-jobs-using-pbs-torque
18+
19+
# otherwise xarray is not available:
20+
module load conda/latest
21+
source /glade/u/apps/opt/conda/bin/activate
22+
23+
# echo ${PBS_JOBID::7}
24+
25+
# Set OpenMP variables
26+
export OMP_NUM_THREADS=1
27+
# export MP_TASK_AFFINITY=core:$OMP_NUM_THREADS
28+
29+
# the easy way
30+
# icar icar_options.nml
31+
32+
# the complex way (allows a continuous sequence of jobs)
33+
PREFIX=$PBS_JOBNAME
34+
35+
# it is useful to keep all other filenames relative to $PREFIX
36+
# note that this is not required anywhere though
37+
OUTDIR=$PREFIX
38+
OPTFILE=${PREFIX}_options.nml
39+
BATCHFILE=${PREFIX}_batch_submit.sh
40+
TEMPLATE=${PREFIX}_template.nml
41+
42+
# specify the location of the icar executable to use:
43+
EXE=${HOME}/bin/icar
44+
45+
# various useful helper scripts (SETUP_RUN is critical)
46+
SETUP_RUN=${HOME}/icar/helpers/setup_next_run.py
47+
MAKE_TEMPLATE=${HOME}/icar/helpers/make_template.py
48+
MKOUTDIR=mkdir #<user_defined_path>/mkscratch.py # mkscratch creates the directory on scratch and links to it
49+
50+
51+
# --------------------------------------------------
52+
# SHOULD NOT NEED TO MODIFY ANYTHING BELOW THIS LINE
53+
# --------------------------------------------------
54+
55+
# if the template file doesn't exist yet, make it
56+
if [[ ! -e $TEMPLATE ]]; then
57+
$MAKE_TEMPLATE $OPTFILE $TEMPLATE > job_output/py_mktemp.out
58+
fi
59+
60+
# if the output directory doesn't exist, create it
61+
if [[ ! -e $OUTDIR ]]; then
62+
$MKOUTDIR $OUTDIR
63+
fi
64+
65+
# if we didn't finish yet we have to continue -BK: but we print this in line 87, so 2 jobs max?
66+
if [[ ! -e ${PREFIX}_finished ]]; then
67+
# first submit the next job dependant on this one
68+
qsub -W depend=afterany:${PBS_JOBID} ${BATCHFILE}
69+
70+
# if we have run before, setup the appropriate restart options
71+
if [[ -e ${PREFIX}_running ]]; then
72+
# echo "setting up next run (setup_next_run.py)"
73+
$SETUP_RUN $OPTFILE $TEMPLATE > job_output/py_setup.out
74+
fi
75+
76+
# declare that we have run before so the next job will know
77+
touch ${PREFIX}_running
78+
79+
# run the actual executable (e.g. icar options.nml)
80+
cafrun -n 36 $EXE $OPTFILE >> job_output/icar${PBS_JOBID::7}.out
81+
# typically the job will get killed while icar is running
82+
# but for some reason bkilling the job still lets it touch _finished...
83+
# maybe this will give it a chance to really kill it first?
84+
sleep 20
85+
86+
# if icar completes, we are done, tell the next job that we finished
87+
touch ${PREFIX}_finished
88+
else
89+
# if the last job ran to completion, delete the inter-job communication files and exit
90+
rm ${PREFIX}_running
91+
rm ${PREFIX}_finished
92+
fi

‎helpers/batch_submit_SLURM.sh‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,97 @@
1+
#!/bin/bash
2+
### Job Name (will be used as prefix later on!)
3+
#SBATCH --job-name="ICAR_tst"
4+
#SBATCH --nodes=1
5+
#SBATCH --ntasks-per-node=32
6+
#SBATCH --time=00:05:00
7+
#SBATCH --constraint=haswell
8+
#SBATCH --qos=debug
9+
### Project code
10+
#SBATCH --account=m4062
11+
### error and output files in separate folder, name with jobid (%x) an job name (%j)
12+
### N.B: create the job_output folder before submitting this job!
13+
#SBATCH --output=job_output/log-%x.%j.out
14+
#SBATCH --error=job_output/log-%x.%j.err
15+
16+
# Make sure a python environment with xarray is available:
17+
module load python
18+
conda activate myenv
19+
20+
# Set OpenMP variables
21+
export OMP_NUM_THREADS=1
22+
# export MP_TASK_AFFINITY=core:$OMP_NUM_THREADS
23+
24+
# the easy way
25+
# icar icar_options.nml
26+
27+
# the complex way (allows a continuous sequence of jobs)
28+
PREFIX=tst ##$SBATCH_JOB_NAME
29+
30+
# it is useful to keep all other filenames relative to $PREFIX
31+
# note that this is not required anywhere though
32+
OUTDIR=$PREFIX
33+
OPTFILE=options.nml #${PREFIX}_options.nml
34+
BATCHFILE=batch_submit_SLURM.sh #${PREFIX}_batch_submit.sh
35+
TEMPLATE=${PREFIX}_template.nml
36+
37+
# the ICAR executable to use
38+
EXE=$HOME/bin/icar_dbs
39+
40+
# load any environmental settings to run icar properly (system dependent):
41+
. /global/cfs/cdirs/m4062/env_scripts/UO-GNU-env.sh
42+
43+
44+
# various useful helper scripts (SETUP_RUN is critical)
45+
SETUP_RUN=${HOME}/icar/helpers/setup_next_run.py
46+
MAKE_TEMPLATE=${HOME}/icar/helpers/make_template.py
47+
MKOUTDIR=mkdir #<user_defined_path>/mkscratch.py # mkscratch creates the directory on scratch and links to it
48+
49+
50+
51+
# --------------------------------------------------
52+
# SHOULD NOT NEED TO MODIFY ANYTHING BELOW THIS LINE
53+
# --------------------------------------------------
54+
55+
# if the template file doesn't exist yet, make it
56+
if [[ ! -e $TEMPLATE ]]; then
57+
$MAKE_TEMPLATE $OPTFILE $TEMPLATE > job_output/py_mktemp.out
58+
fi
59+
60+
# # if the output directory doesn't exist, create it
61+
# if [[ ! -e $OUTDIR ]]; then
62+
# $MKOUTDIR $OUTDIR
63+
# fi
64+
65+
# if we didn't finish yet we have to continue -BK: but we print this in line 87, so 2 jobs max?
66+
if [[ ! -e ${PREFIX}_finished ]]; then
67+
# first submit the next job dependant on this one
68+
# sub -w "ended(${PBS_JOBID})" < $BATCHFILE
69+
# qsub -W depend=afterany:${PBS_JOBID} ${BATCHFILE} ## PBS version
70+
sbatch --dependency=afternotok:$SLURM_JOB_ID ${BATCHFILE}
71+
72+
# if we have run before, setup the appropriate restart options
73+
if [[ -e ${PREFIX}_running ]]; then
74+
# echo "setting up next run (setup_next_run.py)"
75+
$SETUP_RUN $OPTFILE $TEMPLATE > job_output/py_setup.out
76+
fi
77+
78+
# declare that we have run before so the next job will know
79+
touch ${PREFIX}_running
80+
81+
# run the actual executable (e.g. icar options.nml)
82+
# cafrun -n 36 $EXE $OPTFILE > job_output/icar_$SLURM_JOB_ID.out
83+
cafrun -n 36 $EXE $OPTFILE >> job_output/icar.out ### if you prefer one log file for the icar output
84+
85+
# typically the job will get killed while icar is running
86+
# but for some reason bkilling the job still lets it touch _finished...
87+
# maybe this will give it a chance to really kill it first?
88+
sleep 10
89+
90+
# if icar completes, we are done, tell the next job that we finished
91+
# BK dont understand this: wont it prevent the next (or after-next job from starting (ln 63))
92+
touch ${PREFIX}_finished
93+
else
94+
# if the last job ran to completion, delete the inter-job communication files and exit
95+
rm ${PREFIX}_running
96+
rm ${PREFIX}_finished
97+
fi

‎helpers/erai/config.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
def set_bounds(info):
1414
atm_file=info.atmdir+info.atmfile
1515
erai_file=atm_file.replace("_Y_","2000").replace("_M_","01").replace("_D_","01").replace("_h_","00")
16-
varlist=["g4_lat_0","g4_lon_1"]
16+
varlist=["g4_lat_0","g4_lon_1","Z_GDS4_HYBL","T_GDS4_HYBL","Q_GDS4_HYBL","LNSP_GDS4_HYBL","CLWC_GDS4_HYBL","CIWC_GDS4_HYBL","lv_HYBL2_a","lv_HYBL2_b","P0"]
1717
output_dir=info.nc_file_dir
1818
try:
1919
os.mkdir(output_dir)

‎helpers/erai/convert.py‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,29 @@ def convert_atm(data):
3838

3939
return output_data
4040

41+
def bfill(arr):
42+
''' from https://stackoverflow.com/questions/41190852/most-efficient-way-to-forward-fill-nan-values-in-numpy-array
43+
'''
44+
mask = np.isnan(arr)
45+
idx = np.where(~mask, np.arange(mask.shape[1]), mask.shape[1] - 1)
46+
idx = np.minimum.accumulate(idx[:, ::-1], axis=1)[:, ::-1]
47+
out = arr[np.arange(idx.shape[0])[:,None], idx]
48+
return out
49+
50+
51+
def numpy_fill(arr):
52+
'''modified from Solution provided by Divakar.
53+
from https://stackoverflow.com/questions/41190852/most-efficient-way-to-forward-fill-nan-values-in-numpy-array
54+
'''
55+
for i in range(arr.shape[0]):
56+
mask = np.isnan(arr[i])
57+
idx = np.where(~mask,np.arange(mask.shape[1]),0)
58+
np.maximum.accumulate(idx,axis=1, out=idx)
59+
out = arr[i,np.arange(idx.shape[0])[:,None], idx]
60+
arr[i] = bfill(out) # in case there are still missing values on the left side
61+
62+
return arr
63+
4164
# icar_sfc_var=["sensible_heat","latent_heat","hgt_98","PBL_height"]
4265
def convert_sfc(data):
4366
global last_longwave
@@ -53,6 +76,13 @@ def convert_sfc(data):
5376
output_data.lw = data.lw[np.newaxis,::-1,:] / dt # convert from Joules to W /m^2
5477
output_data.cp = data.cp[np.newaxis,::-1,:] * 1000 # convert m to mm
5578

79+
output_data.landmask = data.landmask[np.newaxis,::-1,:]
80+
# landval = data.tskin[np.argmax(data.landmask)] # ~273.15, alternatively, tskin[landmask>0.99].mean()
81+
# above seems to always create an array, and sometimes with very different values in it ... e.g. >300...
82+
landval = 273.16
83+
output_data["sst"] = (data.tskin[np.newaxis,::-1,:] - (output_data.landmask * landval)) / (1 - output_data.landmask)
84+
output_data["sst"][output_data.landmask>0.25] =np.nan
85+
output_data["sst"] = numpy_fill(output_data["sst"])
5686
# this is now handled in io so it can just use the last value in the file, much simple
5787
# ... though in some ways what is below is better as it integrates over a longer time period
5888
# if last_longwave==None:

‎helpers/erai/io_routines.py‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,8 @@
44
from bunch import Bunch
55

66

7-
sfcvarlist=["SSHF_GDS4_SFC","SLHF_GDS4_SFC","Z_GDS4_SFC","BLH_GDS4_SFC","SSRD_GDS4_SFC","STRD_GDS4_SFC", "SKT_GDS4_SFC", "CP_GDS4_SFC"]
8-
icar_sfc_var=["sensible_heat","latent_heat","hgt_98","PBL_height","sw","lw", "tskin", "cp"]
7+
sfcvarlist=["SSHF_GDS4_SFC","SLHF_GDS4_SFC","Z_GDS4_SFC","BLH_GDS4_SFC","SSRD_GDS4_SFC","STRD_GDS4_SFC", "SSTK_GDS4_SFC", "CP_GDS4_SFC", "LSM_GDS4_SFC"]
8+
icar_sfc_var=["sensible_heat","latent_heat","hgt_98","PBL_height","sw","lw", "tskin", "cp","landmask"]
99

1010
atmvarlist=["Z_GDS4_HYBL","T_GDS4_HYBL","Q_GDS4_HYBL","LNSP_GDS4_HYBL","CLWC_GDS4_HYBL","CIWC_GDS4_HYBL","lv_HYBL2_a","lv_HYBL2_b","P0"]
1111
icar_atm_var=["gph","t","qv","ln_p_sfc","cloud","ice","sigma_a","sigma_b","P0"]
@@ -92,7 +92,7 @@ def load_atm(time,info):
9292
"""Load atmospheric variable from a GRIB file"""
9393
uvfile,scfile=find_atm_file(time,info)
9494
uvnc_file=grib2nc(uvfile,atmuvlist,info.nc_file_dir)
95-
scnc_file=grib2nc(scfile,atmvarlist,info.nc_file_dir)
95+
scnc_file=grib2nc(scfile,atmvarlist+["g4_lat_0","g4_lon_1"],info.nc_file_dir)
9696

9797
outputdata=Bunch()
9898
for s,v in zip(icar_uv_var,atmuvlist):

‎helpers/erai/output.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,9 +63,15 @@ def write_file(date,info,erai):
6363
atts=Bunch(long_name="Planetary Boundary Layer Height",units="m")
6464
extra_vars.append(Bunch(name="PBL_height",data=erai["PBL_height"],dims=dims2dt,dtype="f",attributes=atts))
6565

66+
atts=Bunch(long_name="Land fraction",units="")
67+
extra_vars.append(Bunch(name="landfraction",data=erai["landmask"],dims=dims2dt,dtype="f",attributes=atts))
68+
6669
atts=Bunch(long_name="Skin Temperature",units="K")
6770
extra_vars.append(Bunch(name="tskin",data=erai["tskin"],dims=dims2dt,dtype="f",attributes=atts))
6871

72+
atts=Bunch(long_name="Sea Surface Temperature",units="K")
73+
extra_vars.append(Bunch(name="sst",data=erai["sst"],dims=dims2dt,dtype="f",attributes=atts))
74+
6975
atts=Bunch(long_name="Convective precipitation",units="mm")
7076
extra_vars.append(Bunch(name="cp",data=erai["cp"],dims=dims2dt,dtype="f",attributes=atts))
7177

0 commit comments

Comments
 (0)