Skip to content

Commit 90451b0

Browse files
oj-toothoj-tooth
committed
Update functional OceanDataStore CLI tests for Icechunk + Zarr.
1 parent a562c5d commit 90451b0

7 files changed

Lines changed: 52 additions & 39 deletions

‎tests/test_send_to_icechunk_vc_w_dask.slurm‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
# Created On: 2025-05-15
1818
#
1919
# ==============================================================
20-
# -- Input arguments to msm-os -- #
20+
set -euo pipefail
21+
22+
# -- Input arguments to OceanDataStore -- #
2123
# Filepath to eORCA1 ancillary file:
2224
filepath_domain=/dssgfs01/scratch/npd/simulations/Domains/eORCA1/domain_cfg.nc
2325
# Filepath to eORCA1 monthly mean output files:
@@ -31,7 +33,7 @@ dask_config_json=.../OceanDataStore/tests/dask_config.json
3133

3234
# Bucket and object prefix:
3335
bucket=npd-eorca1-era5v1
34-
prefix=test/U1m_icechunk_vc_dask
36+
prefix=test/U1m_icechunk_vc_test_dask
3537

3638
# Define the append dimension:
3739
append_dim=time_counter
@@ -41,15 +43,15 @@ variables="time_counter_bounds uo umo"
4143

4244
# -- Python Environment -- #
4345
# Activate miniconda environment:
44-
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45-
conda activate env_jasmin_os
46+
source /dssgfs01/working/otooth/miniforge3/bin/activate
47+
conda activate env_test
4648

4749
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
4850
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store without Dask..."
4951
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197*.nc
5052
ods send_to_icechunk -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
5153
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
5254
-cs '{"x":360,"y":331,"depthu":25}' -dc $dask_config_json -vc \
53-
-v $variables || { echo "Error: ods send_to_icechunk failed."; exit 1; }
55+
-v $variables
5456

5557
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store without Dask."

‎tests/test_send_to_icechunk_w_dask.slurm‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
# Created On: 2025-05-15
1818
#
1919
# ==============================================================
20-
# -- Input arguments to msm-os -- #
20+
set -euo pipefail
21+
22+
# -- Input arguments to OceanDataStore -- #
2123
# Filepath to eORCA1 ancillary file:
2224
filepath_domain=/dssgfs01/scratch/npd/simulations/Domains/eORCA1/domain_cfg.nc
2325
# Filepath to eORCA1 monthly mean output files:
@@ -37,19 +39,19 @@ prefix=test/U1m_icechunk_dask
3739
append_dim=time_counter
3840

3941
# Define variable:
40-
variables="time_counter_bounds uo umo"
42+
variables="time_counter_bounds uo"
4143

4244
# -- Python Environment -- #
4345
# Activate miniconda environment:
44-
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45-
conda activate env_jasmin_os
46+
source /dssgfs01/working/otooth/miniforge3/bin/activate
47+
conda activate env_test
4648

4749
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
4850
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store without Dask..."
4951
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197*.nc
5052
ods send_to_icechunk -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
5153
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
5254
-cs '{"x":360,"y":331,"depthu":25}' -dc $dask_config_json \
53-
-v $variables || { echo "Error: ods send_to_icechunk failed."; exit 1; }
55+
-v $variables
5456

5557
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store without Dask."

‎tests/test_send_to_icechunk_wo_dask.slurm‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
# Created On: 2025-05-15
1818
#
1919
# ==============================================================
20-
# -- Input arguments to msm-os -- #
20+
set -euo pipefail
21+
22+
# -- Input arguments to OceanDataStore -- #
2123
# Filepath to eORCA1 ancillary file:
2224
filepath_domain=/dssgfs01/scratch/npd/simulations/Domains/eORCA1/domain_cfg.nc
2325
# Filepath to eORCA1 monthly mean output files:
@@ -41,15 +43,15 @@ variables="time_counter_bounds uo umo"
4143

4244
# -- Python Environment -- #
4345
# Activate miniconda environment:
44-
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45-
conda activate env_jasmin_os
46+
source /dssgfs01/working/otooth/miniforge3/bin/activate
47+
conda activate env_test
4648

4749
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
4850
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store without Dask..."
4951
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197*.nc
5052
ods send_to_icechunk -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
5153
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
5254
-cs '{"x":360,"y":331,"depthu":25}' \
53-
-v $variables || { echo "Error: ods send_to_icechunk failed."; exit 1; }
55+
-v $variables
5456

5557
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store without Dask."

‎tests/test_send_to_zarr_w_dask.slurm‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
# Created On: 2025-05-15
1818
#
1919
# ==============================================================
20-
# -- Input arguments to msm-os -- #
20+
set -euo pipefail
21+
22+
# -- Input arguments to OceanDataStore -- #
2123
# Filepath to eORCA1 ancillary file:
2224
filepath_domain=/dssgfs01/scratch/npd/simulations/Domains/eORCA1/domain_cfg.nc
2325
# Filepath to eORCA1 monthly mean output files:
@@ -41,15 +43,15 @@ variables="time_counter_bounds uo umo"
4143

4244
# -- Python Environment -- #
4345
# Activate miniconda environment:
44-
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45-
conda activate env_jasmin_os
46+
source /dssgfs01/working/otooth/miniforge3/bin/activate
47+
conda activate env_test
4648

4749
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
4850
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
4951
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197*.nc
5052
ods send_to_zarr -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
5153
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
5254
-cs '{"x":360,"y":331,"depthu":25}' -v $variables \
53-
-dc $dask_config_json || { echo "Error: ods send failed."; exit 1; }
55+
-dc $dask_config_json
5456

55-
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."
57+
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."

‎tests/test_send_to_zarr_wo_dask.slurm‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
# Created On: 2025-05-15
1818
#
1919
# ==============================================================
20-
# -- Input arguments to msm-os -- #
20+
set -euo pipefail
21+
22+
# -- Input arguments to OceanDataStore -- #
2123
# Filepath to eORCA1 ancillary file:
2224
filepath_domain=/dssgfs01/scratch/npd/simulations/Domains/eORCA1/domain_cfg.nc
2325
# Filepath to eORCA1 monthly mean output files:
@@ -41,15 +43,14 @@ variables="time_counter_bounds uo umo"
4143

4244
# -- Python Environment -- #
4345
# Activate miniconda environment:
44-
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45-
conda activate env_jasmin_os
46+
source /dssgfs01/working/otooth/miniforge3/bin/activate
47+
conda activate env_test
4648

4749
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
4850
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
4951
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197*.nc
5052
ods send_to_zarr -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
5153
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
5254
-cs '{"x":360,"y":331,"depthu":25}' -v $variables \
53-
-dc $dask_config_json || { echo "Error: ods send failed."; exit 1; }
5455

5556
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."

‎tests/test_update_vc_icechunk_w_dask.slurm‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
# Created On: 2025-05-15
1818
#
1919
# ==============================================================
20-
# -- Input arguments to msm-os -- #
20+
set -euo pipefail
21+
22+
# -- Input arguments to OceanDataStore -- #
2123
# Filepath to eORCA1 ancillary file:
2224
filepath_domain=/dssgfs01/scratch/npd/simulations/Domains/eORCA1/domain_cfg.nc
2325
# Filepath to eORCA1 monthly mean output files:
@@ -31,7 +33,7 @@ dask_config_json=.../OceanDataStore/tests/dask_config.json
3133

3234
# Bucket and object prefix:
3335
bucket=npd-eorca1-era5v1
34-
prefix=test/U1m_icechunk_vc_dask
36+
prefix=test/U1m_icechunk_vc_test_dask
3537

3638
# Define the append dimension:
3739
append_dim=time_counter
@@ -41,16 +43,16 @@ variables="time_counter_bounds uo"
4143

4244
# -- Python Environment -- #
4345
# Activate miniconda environment:
44-
source /home/otooth/miniconda3/etc/profile.d/conda.sh
45-
conda activate env_jasmin_os
46+
source /dssgfs01/working/otooth/miniforge3/bin/activate
47+
conda activate env_test
4648

4749
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
4850
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
4951
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197601-197601.nc
5052
ods send_to_icechunk -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
5153
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
5254
-cs '{"x":360,"y":331,"depthu":25}' -v $variables \
53-
-dc $dask_config_json -vc || { echo "Error: ods send_to_icechunk failed."; exit 1; }
55+
-dc $dask_config_json -vc
5456

5557
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."
5658

@@ -59,6 +61,6 @@ filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_1976*.nc
5961
ods update_icechunk -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
6062
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
6163
-ad $append_dim -cs '{"x":360,"y":331,"depthu":25}' -v $variables \
62-
-dc $dask_config_json -vc || { echo "Error: ods update_icechunk failed."; exit 1; }
64+
-dc $dask_config_json
6365

6466
echo "Completed: Updated eORCA1 ERA-5 v1 monthly mean fields in JASMIN object store."

‎tests/test_update_zarr_w_dask.slurm‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717
# Created On: 2025-05-15
1818
#
1919
# ==============================================================
20-
# -- Input arguments to msm-os -- #
20+
set -euo pipefail
21+
22+
# -- Input arguments to OceanDataStore -- #
2123
# Filepath to eORCA1 ancillary file:
2224
filepath_domain=/dssgfs01/scratch/npd/simulations/Domains/eORCA1/domain_cfg.nc
2325

@@ -32,26 +34,26 @@ dask_config_json=.../OceanDataStore/tests/dask_config.json
3234

3335
# Bucket and object prefix:
3436
bucket=npd-eorca1-era5v1
35-
prefix=test/U1m_zarr_dask
37+
prefix=test/U1m_zarr_update_dask
3638

3739
# Define the append dimension:
3840
append_dim=time_counter
3941

4042
# Define variable:
41-
variables="time_counter_bounds uo"
43+
variables="time_counter_bounds uo umo"
4244

4345
# -- Python Environment -- #
4446
# Activate miniconda environment:
45-
source /home/otooth/miniconda3/etc/profile.d/conda.sh
46-
conda activate env_jasmin_os
47+
source /dssgfs01/working/otooth/miniforge3/bin/activate
48+
conda activate env_test
4749

4850
# -- Send eORCA1 ERA-5 v1 monthly mean outputs to JASMIN OS -- #
4951
echo "In Progress: Sending eORCA1 ERA-5 v1 U1m variables to JASMIN object store..."
50-
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197601-197601.nc
52+
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_1976*.nc
5153
ods send_to_zarr -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
5254
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
53-
-ad $append_dim -cs '{"x":360,"y":331,"depthu":25}' -v $variables \
54-
-dc $dask_config_json|| { echo "Error: ods send_to_zarr failed."; exit 1; }
55+
-cs '{"x":360,"y":331,"depthu":25}' -v $variables \
56+
-dc $dask_config_json
5557

5658
echo "Completed: Sent eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."
5759

@@ -60,7 +62,7 @@ echo "In Progress: Updating eORCA1 ERA-5 v1 U1m variables to JASMIN object store
6062
filepath_gridU=$filedir/eORCA1_ERA5_1m_grid_U_197*.nc
6163
ods update_zarr -f $filepath_gridU -c $store_credentials_json -b $bucket -p $prefix \
6264
-gf $filepath_domain -uc '{"nav_lon":"glamu", "nav_lat":"gphiu"}' \
63-
-ad $append_dim -cs '{"x":360,"y":331,"depthu":25}' -v $variables \
64-
-dc $dask_config_json || { echo "Error: ods update_zarr failed."; exit 1; }
65+
-cs '{"x":360,"y":331,"depthu":25}' -v $variables \
66+
-dc $dask_config_json
6567

6668
echo "Completed: Updated eORCA1 ERA-5 v1 monthly mean fields to JASMIN object store."

0 commit comments

Comments
 (0)