Skip to content

Commit 0d327b2

Browse files
seanlawseanlaw
authored
Fixed #260 Split Unit & Coverage Tests (#261)
* Fixed #260 Split Unit & Coverage Tests * Swapped T_A/T_B in GPU tests for performance * Updated displayNames in Azure Pipelines
1 parent 06d7530 commit 0d327b2

4 files changed

Lines changed: 278 additions & 161 deletions

File tree

‎azure-pipelines.yml‎

Lines changed: 66 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,10 @@ pr:
1010
- master
1111

1212
stages:
13-
- stage: Build
13+
- stage: Unit
1414

1515
jobs:
16-
- job: Job
16+
- job: Test
1717

1818
strategy:
1919
matrix:
@@ -68,12 +68,73 @@ stages:
6868
flake8 ./
6969
displayName: "Running flake8"
7070
71-
- bash: ./test.sh
72-
displayName: "Running test"
71+
- bash: ./test.sh unit
72+
displayName: "Running unit tests"
73+
74+
- stage: Coverage
75+
76+
jobs:
77+
- job: Test
78+
79+
strategy:
80+
matrix:
81+
python36_linux:
82+
imageName: "ubuntu-16.04"
83+
python.version: "3.6"
84+
python36_windows:
85+
imageName: "vs2017-win2016"
86+
python.version: "3.6"
87+
python36_macos:
88+
imageName: "macos-10.14"
89+
python.version: "3.6"
90+
python37_linux:
91+
imageName: "ubuntu-16.04"
92+
python.version: "3.7"
93+
python37_windows:
94+
imageName: "vs2017-win2016"
95+
python.version: "3.7"
96+
python37_macos:
97+
imageName: "macos-10.14"
98+
python.version: "3.7"
99+
python38_linux:
100+
imageName: "ubuntu-16.04"
101+
python.version: "3.8"
102+
python38_windows:
103+
imageName: "vs2017-win2016"
104+
python.version: "3.8"
105+
python38_macos:
106+
imageName: "macos-10.14"
107+
python.version: "3.8"
108+
maxParallel: 10
109+
110+
pool:
111+
vmImage: $(imageName)
112+
113+
steps:
114+
- task: UsePythonVersion@0
115+
inputs:
116+
versionSpec: '$(python.version)'
117+
displayName: 'Use Python $(python.version)'
118+
119+
- script: |
120+
pip install --editable .[ci]
121+
pip install pytest-azurepipelines
122+
displayName: 'Install stumpy and other dependencies'
123+
124+
- script: |
125+
black --check --diff ./
126+
displayName: "Running black"
127+
128+
- script: |
129+
flake8 ./
130+
displayName: "Running flake8"
131+
132+
- bash: ./test.sh coverage
133+
displayName: "Running test coverage"
73134

74135
- script: |
75136
codecov --token=$(CODECOV_TOKEN)
76-
displayName: "Upload test coverage results"
137+
displayName: "Uploading test coverage results"
77138
78139
- stage: Scan
79140
displayName: "FOSSA"

‎test.sh‎

Lines changed: 157 additions & 101 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,19 @@
1-
#!/bin/sh
1+
#!/bin/bash
2+
3+
test_mode="all"
4+
5+
# Parse first command line argument
6+
if [ $# -gt 0 ]; then
7+
if [ $1 == "unit" ]; then
8+
test_mode="unit"
9+
elif [ $1 == "coverage" ]; then
10+
test_mode="coverage"
11+
elif [ $1 == "custom" ]; then
12+
test_mode="custom"
13+
else
14+
echo "Using default test_mode=\"all\""
15+
fi
16+
fi
217

318
###############
419
# Functions #
@@ -14,105 +29,146 @@ check_errs()
1429
fi
1530
}
1631

17-
echo "Checking Black Code Formatting"
18-
black --check --diff ./
19-
check_errs $?
20-
21-
echo "Checking Flake8 Style Guide Enforcement"
22-
flake8 ./
23-
check_errs $?
24-
25-
# Test a single function repeatedly
26-
if false; then
27-
# if true; then
28-
# echo "Disabling Numba JIT and CUDA Compiled Functions"
29-
# export NUMBA_DISABLE_JIT=1
30-
# export NUMBA_ENABLE_CUDASIM=1
31-
for VARIABLE in {1..10}
32-
do
33-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_.py
34-
check_errs $?
35-
done
36-
exit 0
32+
check_black()
33+
{
34+
echo "Checking Black Code Formatting"
35+
black --check --diff ./
36+
check_errs $?
37+
}
38+
39+
check_flake()
40+
{
41+
echo "Checking Flake8 Style Guide Enforcement"
42+
flake8 ./
43+
check_errs $?
44+
}
45+
46+
test_custom()
47+
{
48+
# Test one or more user-defined functions repeatedly
49+
for VARIABLE in {1..10}
50+
do
51+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_.py
52+
check_errs $?
53+
done
54+
clean_up
55+
exit 0
56+
}
57+
58+
test_unit()
59+
{
60+
echo "Testing Numba JIT Compiled Functions"
61+
py.test -rsx -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_gpu_stump.py
62+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_core.py
63+
check_errs $?
64+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stump.py tests/test_mstump.py tests/test_scrump.py tests/test_stumpi.py
65+
check_errs $?
66+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped.py
67+
check_errs $?
68+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_one_constant_subsequence.py
69+
check_errs $?
70+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_constant_subsequences.py
71+
check_errs $?
72+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_constant_subsequences_swap.py
73+
check_errs $?
74+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_one_subsequence_nan_self_join.py
75+
check_errs $?
76+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_one_subsequence_inf_self_join.py
77+
check_errs $?
78+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_one_subsequence_nan_A_B_join.py
79+
check_errs $?
80+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_one_subsequence_inf_A_B_join.py
81+
check_errs $?
82+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_subsequences_nan_A_B_join.py
83+
check_errs $?
84+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_subsequences_inf_A_B_join.py
85+
check_errs $?
86+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_subsequences_nan_inf_A_B_join.py
87+
check_errs $?
88+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_subsequences_nan_inf_A_B_join_swap.py
89+
check_errs $?
90+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_mstumped.py
91+
check_errs $?
92+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_mstumped_one_subsequence_nan_self_join.py
93+
check_errs $?
94+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_mstumped_one_subsequence_nan_self_join.py
95+
check_errs $?
96+
# aamp
97+
py.test -rsx -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_gpu_aamp.py
98+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamp.py tests/test_aampi.py
99+
check_errs $?
100+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped.py
101+
check_errs $?
102+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_one_constant_subsequence.py
103+
check_errs $?
104+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_constant_subsequences.py
105+
check_errs $?
106+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_constant_subsequences_swap.py
107+
check_errs $?
108+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_one_subsequence_nan_self_join.py
109+
check_errs $?
110+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_one_subsequence_inf_self_join.py
111+
check_errs $?
112+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_one_subsequence_nan_A_B_join.py
113+
check_errs $?
114+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_one_subsequence_inf_A_B_join.py
115+
check_errs $?
116+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_subsequences_nan_A_B_join.py
117+
check_errs $?
118+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_subsequences_inf_A_B_join.py
119+
check_errs $?
120+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_subsequences_nan_inf_A_B_join.py
121+
check_errs $?
122+
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_subsequences_nan_inf_A_B_join_swap.py
123+
check_errs $?
124+
}
125+
126+
test_coverage()
127+
{
128+
echo "Disabling Numba JIT and CUDA Compiled Functions"
129+
export NUMBA_DISABLE_JIT=1
130+
export NUMBA_ENABLE_CUDASIM=1
131+
132+
# echo "Testing Python Functions"
133+
# py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests
134+
# check_errs $?
135+
136+
echo "Testing Code Coverage"
137+
coverage run --source stumpy -m py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning
138+
check_errs $?
139+
coverage report -m
140+
}
141+
142+
clean_up()
143+
{
144+
echo "Cleaning Up"
145+
rm -rf "dask-worker-space"
146+
}
147+
148+
###########
149+
# Main #
150+
###########
151+
152+
check_black
153+
check_flake
154+
155+
if [ $test_mode == "unit" ]; then
156+
echo "Executing Unit Tests Only"
157+
test_unit
158+
elif [ $test_mode == "coverage" ]; then
159+
echo "Executing Code Coverage Only"
160+
test_coverage
161+
elif [ $test_mode == "custom" ]; then
162+
echo "Executing Custom User-Defined Tests Only"
163+
# Define tests in `test_custom` function above
164+
# echo "Disabling Numba JIT and CUDA Compiled Functions"
165+
# export NUMBA_DISABLE_JIT=1
166+
# export NUMBA_ENABLE_CUDASIM=1
167+
test_custom
168+
else
169+
echo "Executing Unit Tests And Code Coverage"
170+
test_unit
171+
test_coverage
37172
fi
38173

39-
echo "Testing Numba JIT Compiled Functions"
40-
py.test -rsx -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_gpu_stump.py
41-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_core.py
42-
check_errs $?
43-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stump.py tests/test_mstump.py tests/test_scrump.py tests/test_stumpi.py
44-
check_errs $?
45-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped.py
46-
check_errs $?
47-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_one_constant_subsequence.py
48-
check_errs $?
49-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_constant_subsequences.py
50-
check_errs $?
51-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_constant_subsequences_swap.py
52-
check_errs $?
53-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_one_subsequence_nan_self_join.py
54-
check_errs $?
55-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_one_subsequence_inf_self_join.py
56-
check_errs $?
57-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_one_subsequence_nan_A_B_join.py
58-
check_errs $?
59-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_one_subsequence_inf_A_B_join.py
60-
check_errs $?
61-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_subsequences_nan_A_B_join.py
62-
check_errs $?
63-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_subsequences_inf_A_B_join.py
64-
check_errs $?
65-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_subsequences_nan_inf_A_B_join.py
66-
check_errs $?
67-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_stumped_two_subsequences_nan_inf_A_B_join_swap.py
68-
check_errs $?
69-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_mstumped.py
70-
check_errs $?
71-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_mstumped_one_subsequence_nan_self_join.py
72-
check_errs $?
73-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_mstumped_one_subsequence_nan_self_join.py
74-
check_errs $?
75-
# aamp
76-
py.test -rsx -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_gpu_aamp.py
77-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamp.py tests/test_aampi.py
78-
check_errs $?
79-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped.py
80-
check_errs $?
81-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_one_constant_subsequence.py
82-
check_errs $?
83-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_constant_subsequences.py
84-
check_errs $?
85-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_constant_subsequences_swap.py
86-
check_errs $?
87-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_one_subsequence_nan_self_join.py
88-
check_errs $?
89-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_one_subsequence_inf_self_join.py
90-
check_errs $?
91-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_one_subsequence_nan_A_B_join.py
92-
check_errs $?
93-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_one_subsequence_inf_A_B_join.py
94-
check_errs $?
95-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_subsequences_nan_A_B_join.py
96-
check_errs $?
97-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_subsequences_inf_A_B_join.py
98-
check_errs $?
99-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_subsequences_nan_inf_A_B_join.py
100-
check_errs $?
101-
py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests/test_aamped_two_subsequences_nan_inf_A_B_join_swap.py
102-
check_errs $?
103-
104-
echo "Disabling Numba JIT and CUDA Compiled Functions"
105-
export NUMBA_DISABLE_JIT=1
106-
export NUMBA_ENABLE_CUDASIM=1
107-
108-
# echo "Testing Python Functions"
109-
# py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning tests
110-
# check_errs $?
111-
112-
echo "Test Python Functions and Code Coverage"
113-
coverage run --source stumpy -m py.test -x -W ignore::RuntimeWarning -W ignore::DeprecationWarning
114-
check_errs $?
115-
coverage report -m
116-
117-
echo "Cleaning Up"
118-
rm -rf "dask-worker-space"
174+
clean_up

0 commit comments

Comments
 (0)