Skip to content
This repository was archived by the owner on May 1, 2024. It is now read-only.

Commit 8c53e77

Browse files
muhammad-ammarmuhammad-ammar
authored
Merge pull request #710 from openedx/ammar/streaming-csv
feat: add streaming csv support
2 parents ddca804 + acbf865 commit 8c53e77

9 files changed

Lines changed: 51 additions & 38 deletions

File tree

‎analyticsdataserver/settings/base.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -358,6 +358,7 @@
358358
ANALYTICS_DATABASE_V1 = None
359359
DATABASE_ROUTERS = ['analyticsdataserver.router.AnalyticsAPIRouter', 'analyticsdataserver.router.AnalyticsModelsRouter']
360360
ENTERPRISE_REPORTING_DB_ALIAS = 'enterprise'
361+
ENROLLMENTS_PAGE_SIZE = 10000
361362

362363
LMS_BASE_URL = None
363364

‎analyticsdataserver/settings/local.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
ANALYTICS_DATABASE = 'analytics'
7676
ENTERPRISE_REPORTING_DB_ALIAS = 'analytics'
7777
ANALYTICS_DATABASE_V1 = 'analytics'
78+
ENROLLMENTS_PAGE_SIZE = 10000
7879

7980
TEST_RUNNER = 'django_nose.NoseTestSuiteRunner'
8081

‎analyticsdataserver/settings/test.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
ANALYTICS_DATABASE = 'analytics'
3535
ANALYTICS_DATABASE_V1 = 'analytics_v1'
3636
ENTERPRISE_REPORTING_DB_ALIAS = 'default'
37+
ENROLLMENTS_PAGE_SIZE = 10000
3738

3839
LMS_BASE_URL = 'http://lms-host'
3940

‎requirements/base.txt‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ backports-zoneinfo==0.2.1
1313
# via django
1414
boto==2.49.0
1515
# via -r requirements/base.in
16-
boto3==1.34.35
16+
boto3==1.34.40
1717
# via -r requirements/base.in
18-
botocore==1.34.35
18+
botocore==1.34.40
1919
# via
2020
# boto3
2121
# s3transfer
@@ -69,7 +69,7 @@ django-fernet-fields-v2==0.9
6969
# via edx-enterprise-data
7070
django-filter==23.5
7171
# via edx-enterprise-data
72-
django-model-utils==4.3.1
72+
django-model-utils==4.4.0
7373
# via
7474
# edx-enterprise-data
7575
# edx-rbac
@@ -90,7 +90,9 @@ djangorestframework==3.14.0
9090
# edx-api-doc-tools
9191
# edx-drf-extensions
9292
djangorestframework-csv==3.0.2
93-
# via -r requirements/base.in
93+
# via
94+
# -r requirements/base.in
95+
# edx-enterprise-data
9496
drf-jwt==1.19.2
9597
# via edx-drf-extensions
9698
drf-yasg==1.21.7
@@ -112,7 +114,7 @@ edx-drf-extensions==10.2.0
112114
# -r requirements/base.in
113115
# edx-enterprise-data
114116
# edx-rbac
115-
edx-enterprise-data==5.5.1
117+
edx-enterprise-data==6.0.0
116118
# via -r requirements/base.in
117119
edx-opaque-keys==2.5.1
118120
# via
@@ -128,7 +130,7 @@ edx-rest-api-client==5.6.1
128130
# edx-enterprise-data
129131
factory-boy==3.3.0
130132
# via edx-enterprise-data
131-
faker==22.7.0
133+
faker==23.1.0
132134
# via factory-boy
133135
html5lib==1.1
134136
# via -r requirements/base.in
@@ -215,7 +217,7 @@ stevedore==5.1.0
215217
# via
216218
# edx-django-utils
217219
# edx-opaque-keys
218-
tqdm==4.66.1
220+
tqdm==4.66.2
219221
# via -r requirements/base.in
220222
typing-extensions==4.9.0
221223
# via

‎requirements/dev.txt‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ backports-zoneinfo==0.2.1
1313
# via django
1414
boto==2.49.0
1515
# via -r requirements/base.in
16-
boto3==1.34.35
16+
boto3==1.34.40
1717
# via -r requirements/base.in
18-
botocore==1.34.35
18+
botocore==1.34.40
1919
# via
2020
# boto3
2121
# s3transfer
@@ -69,7 +69,7 @@ django-fernet-fields-v2==0.9
6969
# via edx-enterprise-data
7070
django-filter==23.5
7171
# via edx-enterprise-data
72-
django-model-utils==4.3.1
72+
django-model-utils==4.4.0
7373
# via
7474
# edx-enterprise-data
7575
# edx-rbac
@@ -90,7 +90,9 @@ djangorestframework==3.14.0
9090
# edx-api-doc-tools
9191
# edx-drf-extensions
9292
djangorestframework-csv==3.0.2
93-
# via -r requirements/base.in
93+
# via
94+
# -r requirements/base.in
95+
# edx-enterprise-data
9496
drf-jwt==1.19.2
9597
# via edx-drf-extensions
9698
drf-yasg==1.21.7
@@ -112,7 +114,7 @@ edx-drf-extensions==10.2.0
112114
# -r requirements/base.in
113115
# edx-enterprise-data
114116
# edx-rbac
115-
edx-enterprise-data==5.5.1
117+
edx-enterprise-data==6.0.0
116118
# via -r requirements/base.in
117119
edx-opaque-keys==2.5.1
118120
# via
@@ -128,7 +130,7 @@ edx-rest-api-client==5.6.1
128130
# edx-enterprise-data
129131
factory-boy==3.3.0
130132
# via edx-enterprise-data
131-
faker==22.7.0
133+
faker==23.1.0
132134
# via factory-boy
133135
html5lib==1.1
134136
# via -r requirements/base.in
@@ -150,7 +152,7 @@ markdown==3.5.2
150152
# via -r requirements/base.in
151153
markupsafe==2.1.5
152154
# via jinja2
153-
mysqlclient==2.2.3
155+
mysqlclient==2.2.4
154156
# via -r requirements/dev.in
155157
newrelic==9.6.0
156158
# via edx-django-utils
@@ -217,7 +219,7 @@ stevedore==5.1.0
217219
# via
218220
# edx-django-utils
219221
# edx-opaque-keys
220-
tqdm==4.66.1
222+
tqdm==4.66.2
221223
# via -r requirements/base.in
222224
typing-extensions==4.9.0
223225
# via

‎requirements/doc.txt‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@ beautifulsoup4==4.12.3
2323
# via pydata-sphinx-theme
2424
boto==2.49.0
2525
# via -r requirements/base.in
26-
boto3==1.34.35
26+
boto3==1.34.40
2727
# via -r requirements/base.in
28-
botocore==1.34.35
28+
botocore==1.34.40
2929
# via
3030
# boto3
3131
# s3transfer
@@ -79,7 +79,7 @@ django-fernet-fields-v2==0.9
7979
# via edx-enterprise-data
8080
django-filter==23.5
8181
# via edx-enterprise-data
82-
django-model-utils==4.3.1
82+
django-model-utils==4.4.0
8383
# via
8484
# edx-enterprise-data
8585
# edx-rbac
@@ -100,7 +100,9 @@ djangorestframework==3.14.0
100100
# edx-api-doc-tools
101101
# edx-drf-extensions
102102
djangorestframework-csv==3.0.2
103-
# via -r requirements/base.in
103+
# via
104+
# -r requirements/base.in
105+
# edx-enterprise-data
104106
docutils==0.19
105107
# via
106108
# pydata-sphinx-theme
@@ -126,7 +128,7 @@ edx-drf-extensions==10.2.0
126128
# -r requirements/base.in
127129
# edx-enterprise-data
128130
# edx-rbac
129-
edx-enterprise-data==5.5.1
131+
edx-enterprise-data==6.0.0
130132
# via -r requirements/base.in
131133
edx-opaque-keys==2.5.1
132134
# via
@@ -142,7 +144,7 @@ edx-rest-api-client==5.6.1
142144
# edx-enterprise-data
143145
factory-boy==3.3.0
144146
# via edx-enterprise-data
145-
faker==22.7.0
147+
faker==23.1.0
146148
# via factory-boy
147149
html5lib==1.1
148150
# via -r requirements/base.in
@@ -272,7 +274,7 @@ stevedore==5.1.0
272274
# via
273275
# edx-django-utils
274276
# edx-opaque-keys
275-
tqdm==4.66.1
277+
tqdm==4.66.2
276278
# via -r requirements/base.in
277279
typing-extensions==4.9.0
278280
# via

‎requirements/pip.txt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,5 +10,5 @@ wheel==0.42.0
1010
# The following packages are considered to be unsafe in a requirements file:
1111
pip==24.0
1212
# via -r requirements/pip.in
13-
setuptools==69.0.3
13+
setuptools==69.1.0
1414
# via -r requirements/pip.in

‎requirements/production.txt‎

Lines changed: 10 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -13,9 +13,9 @@ backports-zoneinfo==0.2.1
1313
# via django
1414
boto==2.49.0
1515
# via -r requirements/base.in
16-
boto3==1.34.35
16+
boto3==1.34.40
1717
# via -r requirements/base.in
18-
botocore==1.34.35
18+
botocore==1.34.40
1919
# via
2020
# boto3
2121
# s3transfer
@@ -69,7 +69,7 @@ django-fernet-fields-v2==0.9
6969
# via edx-enterprise-data
7070
django-filter==23.5
7171
# via edx-enterprise-data
72-
django-model-utils==4.3.1
72+
django-model-utils==4.4.0
7373
# via
7474
# edx-enterprise-data
7575
# edx-rbac
@@ -90,7 +90,9 @@ djangorestframework==3.14.0
9090
# edx-api-doc-tools
9191
# edx-drf-extensions
9292
djangorestframework-csv==3.0.2
93-
# via -r requirements/base.in
93+
# via
94+
# -r requirements/base.in
95+
# edx-enterprise-data
9496
drf-jwt==1.19.2
9597
# via edx-drf-extensions
9698
drf-yasg==1.21.7
@@ -112,7 +114,7 @@ edx-drf-extensions==10.2.0
112114
# -r requirements/base.in
113115
# edx-enterprise-data
114116
# edx-rbac
115-
edx-enterprise-data==5.5.1
117+
edx-enterprise-data==6.0.0
116118
# via -r requirements/base.in
117119
edx-opaque-keys==2.5.1
118120
# via
@@ -128,7 +130,7 @@ edx-rest-api-client==5.6.1
128130
# edx-enterprise-data
129131
factory-boy==3.3.0
130132
# via edx-enterprise-data
131-
faker==22.7.0
133+
faker==23.1.0
132134
# via factory-boy
133135
gevent==23.9.1
134136
# via -r requirements/production.in
@@ -156,7 +158,7 @@ markdown==3.5.2
156158
# via -r requirements/base.in
157159
markupsafe==2.1.5
158160
# via jinja2
159-
mysqlclient==2.2.3
161+
mysqlclient==2.2.4
160162
# via -r requirements/production.in
161163
newrelic==9.6.0
162164
# via
@@ -230,7 +232,7 @@ stevedore==5.1.0
230232
# via
231233
# edx-django-utils
232234
# edx-opaque-keys
233-
tqdm==4.66.1
235+
tqdm==4.66.2
234236
# via -r requirements/base.in
235237
typing-extensions==4.9.0
236238
# via

‎requirements/test.txt‎

Lines changed: 9 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@ backports-zoneinfo==0.2.1
1515
# via django
1616
boto==2.49.0
1717
# via -r requirements/base.in
18-
boto3==1.34.35
18+
boto3==1.34.40
1919
# via -r requirements/base.in
20-
botocore==1.34.35
20+
botocore==1.34.40
2121
# via
2222
# boto3
2323
# s3transfer
@@ -82,7 +82,7 @@ django-fernet-fields-v2==0.9
8282
# via edx-enterprise-data
8383
django-filter==23.5
8484
# via edx-enterprise-data
85-
django-model-utils==4.3.1
85+
django-model-utils==4.4.0
8686
# via
8787
# edx-enterprise-data
8888
# edx-rbac
@@ -103,7 +103,9 @@ djangorestframework==3.14.0
103103
# edx-api-doc-tools
104104
# edx-drf-extensions
105105
djangorestframework-csv==3.0.2
106-
# via -r requirements/base.in
106+
# via
107+
# -r requirements/base.in
108+
# edx-enterprise-data
107109
drf-jwt==1.19.2
108110
# via edx-drf-extensions
109111
drf-yasg==1.21.7
@@ -125,7 +127,7 @@ edx-drf-extensions==10.2.0
125127
# -r requirements/base.in
126128
# edx-enterprise-data
127129
# edx-rbac
128-
edx-enterprise-data==5.5.1
130+
edx-enterprise-data==6.0.0
129131
# via -r requirements/base.in
130132
edx-opaque-keys==2.5.1
131133
# via
@@ -143,7 +145,7 @@ exceptiongroup==1.2.0
143145
# via pytest
144146
factory-boy==3.3.0
145147
# via edx-enterprise-data
146-
faker==22.7.0
148+
faker==23.1.0
147149
# via factory-boy
148150
freezegun==1.4.0
149151
# via -r requirements/test.in
@@ -279,7 +281,7 @@ tomli==2.0.1
279281
# via
280282
# coverage
281283
# pytest
282-
tqdm==4.66.1
284+
tqdm==4.66.2
283285
# via -r requirements/base.in
284286
typing-extensions==4.9.0
285287
# via

0 commit comments

Comments
 (0)