Adds CSV support to the LearnerListView

* Puts pagination previous/next links in the HTTP header
* Allows the returned field list (for CSV only) to be limited by the
  'fields' query string parameter.
* Flattens CSV list fields into a comma-delimited string, instead of showing list.0,list.1..
* Upgrades djangorestframework-csv to maintain fields/heading sort order in CSV.
* Avoids duplicate-code warnings by moving the CSV response validation tests to
  a new analytics_data_api.v0.tests.views.VerifyCsvResponseMixin

Author: pomegranited

analytics_data_api/constants/learner.py

@@ -1,3 +1 @@
-LEARNER_API_DEFAULT_LIST_PAGE_SIZE = 25
-
 SEGMENTS = ["highly_engaged", "disengaging", "struggling", "inactive", "unenrolled"]

analytics_data_api/renderers.py

@@ -0,0 +1,96 @@
+"""
+Custom REST framework renderers common to all versions of the API.
+"""
+from rest_framework_csv.renderers import CSVRenderer
+from ordered_set import OrderedSet
+
+
+class ResultsOnlyRendererMixin(object):
+    """
+    Render data using just the results array.
+
+    Use with PaginatedHeadersMixin to preserve the pagination links in the response header.
+    """
+    results_field = 'results'
+
+    def render(self, data, *args, **kwargs):
+        """
+        Replace the rendered data with just what is in the results_field.
+        """
+        if not isinstance(data, list):
+            data = data.get(self.results_field, [])
+        return super(ResultsOnlyRendererMixin, self).render(data, *args, **kwargs)
+
+
+class DynamicFieldsCsvRenderer(CSVRenderer):
+    """
+    Allows the `fields` query parameter to determine which fields should be
+    returned in the response, and in what order.
+
+    Note that if no header is provided, and the fields_param query string
+    parameter is not found in the request, the fields are rendered in
+    alphabetical order.
+    """
+    # Name of the query string parameter to check for the fields list
+    # Set to None to ensure that any request fields will not override
+    fields_param = 'fields'
+
+    # Seperator character(s) to split the fields parameter list
+    fields_sep = ','
+
+    # Set to None to flatten lists into one heading per value.
+    # Otherwise, concatenate lists delimiting with the given string.
+    concatenate_lists_sep = ', '
+
+    def flatten_list(self, l):
+        if self.concatenate_lists_sep is None:
+            return super(DynamicFieldsCsvRenderer, self).flatten_list(l)
+        return {'': self.concatenate_lists_sep.join(l)}
+
+    def get_header(self, data, renderer_context):
+        """Return the list of header fields, determined by class settings and context."""
+
+        # Start with the previously-set list of header fields
+        header = renderer_context.get('header', self.header)
+
+        # If no previous set, then determine the candidates from the data
+        if header is None:
+            header = set()
+            data = self.flatten_data(data)
+            for item in data:
+                header.update(list(item.keys()))
+
+            # Alphabetize header fields by default, since
+            # flatten_data() makes field order indeterminate.
+            header = sorted(header)
+
+        # If configured to, examine the query parameters for the requsted header fields
+        request = renderer_context.get('request')
+        if request is not None and self.fields_param is not None:
+
+            request_fields = request.query_params.get(self.fields_param)
+            if request_fields is not None:
+
+                requested = OrderedSet()
+                for request_field in request_fields.split(self.fields_sep):
+
+                    # Only fields in the original candidate header set are valid
+                    if request_field in header:
+                        requested.update((request_field,))
+
+                header = requested  # pylint: disable=redefined-variable-type
+
+        return header
+
+    def render(self, data, media_type=None, renderer_context=None, writer_opts=None):
+        """Override the default "get headers" behaviour, then render the data."""
+        renderer_context = renderer_context or {}
+        self.header = self.get_header(data, renderer_context)
+        return super(DynamicFieldsCsvRenderer, self).render(data, media_type, renderer_context, writer_opts)
+
+
+class PaginatedCsvRenderer(ResultsOnlyRendererMixin, DynamicFieldsCsvRenderer):
+    """
+    Render results-only CSV data with dynamically-determined fields.
+    """
+    media_type = 'text/csv'

analytics_data_api/tests/__init__.py

@@ -0,0 +1,77 @@
+"""
+Tests for the custom REST framework renderers.
+"""
+from mock import MagicMock, PropertyMock
+from django.test import TestCase
+
+from analytics_data_api.renderers import PaginatedCsvRenderer
+
+
+class PaginatedCsvRendererTests(TestCase):
+
+    def setUp(self):
+        super(PaginatedCsvRendererTests, self).setUp()
+        self.renderer = PaginatedCsvRenderer()
+        self.data = {'results': [
+            {
+                'string': 'ab,c',
+                'list': ['a', 'b', 'c'],
+                'dict': {'a': 1, 'b': 2, 'c': 3},
+            }, {
+                'string': 'def',
+                'string2': 'ghi',
+                'list': ['d', 'e', 'f', 'g'],
+                'dict': {'d': 4, 'b': 5, 'c': 6},
+            },
+        ]}
+        self.context = {}
+
+    def set_request(self, params=None):
+        request = MagicMock()
+        mock_params = PropertyMock(return_value=params)
+        type(request).query_params = mock_params
+        self.context['request'] = request
+
+    def test_csv_media_type(self):
+        self.assertEqual(self.renderer.media_type, 'text/csv')
+
+    def test_render(self):
+        rendered_data = self.renderer.render(self.data, renderer_context=self.context)
+        self.assertEquals(rendered_data,
+                          'dict.a,dict.b,dict.c,dict.d,list,string,string2\r\n'
+                          '1,2,3,,"a, b, c","ab,c",\r\n'
+                          ',5,6,4,"d, e, f, g",def,ghi\r\n')
+
+    def test_render_fields(self):
+        self.set_request(dict(fields='string2,invalid,dict.b,list,dict.a,string'))
+        rendered_data = self.renderer.render(self.data, renderer_context=self.context)
+        self.assertEquals(rendered_data,
+                          'string2,dict.b,list,dict.a,string\r\n'
+                          ',2,"a, b, c",1,"ab,c"\r\n'
+                          'ghi,5,"d, e, f, g",,def\r\n')
+
+    def test_render_flatten_lists(self):
+        self.renderer.concatenate_lists_sep = None
+        rendered_data = self.renderer.render(self.data, renderer_context=self.context)
+        self.assertEquals(rendered_data,
+                          'dict.a,dict.b,dict.c,dict.d,list.0,list.1,list.2,list.3,string,string2\r\n'
+                          '1,2,3,,a,b,c,,"ab,c",\r\n'
+                          ',5,6,4,d,e,f,g,def,ghi\r\n')
+
+    def test_render_fields_flatten_lists(self):
+        self.renderer.concatenate_lists_sep = None
+        self.set_request(dict(fields='string2,invalid,list.2,dict.a,list.1,string'))
+        rendered_data = self.renderer.render(self.data, renderer_context=self.context)
+        self.assertEquals(rendered_data,
+                          'string2,list.2,dict.a,list.1,string\r\n'
+                          ',c,1,b,"ab,c"\r\n'
+                          'ghi,f,,e,def\r\n')
+
+    def test_render_fields_limit_headers(self):
+        self.renderer.header = ('string2', 'invalid', 'dict.a')
+        self.set_request(dict(fields='string2,invalid,dict.b,list,dict.a,string'))
+        rendered_data = self.renderer.render(self.data, renderer_context=self.context)
+        self.assertEquals(rendered_data,
+                          'string2,invalid,dict.a\r\n'
+                          ',,1\r\n'
+                          'ghi,,\r\n')

analytics_data_api/tests/test_renderers.py

@@ -7,7 +7,6 @@
 from analytics_data_api.constants import (
     engagement_events,
     enrollment_modes,
-    learner,
 )
 from analytics_data_api.v0 import models
 
@@ -409,8 +408,8 @@ class EdxPaginationSerializer(pagination.PageNumberPagination):
     Adds values to the response according to edX REST API Conventions.
     """
     page_size_query_param = 'page_size'
-    page_size = learner.LEARNER_API_DEFAULT_LIST_PAGE_SIZE
-    max_page_size = 100  # TODO -- tweak during load testing
+    page_size = getattr(settings, 'DEFAULT_PAGE_SIZE', 25)
+    max_page_size = getattr(settings, 'MAX_PAGE_SIZE', 100)  # TODO -- tweak during load testing
 
     def get_paginated_response(self, data):
         # The output is more readable with num_pages included not at the end, but

analytics_data_api/tests.py analytics_data_api/tests/test_utils.pyanalytics_data_api/tests.py renamed to analytics_data_api/tests/test_utils.py

@@ -1,9 +1,13 @@
 import json
+import StringIO
+import csv
 
 from opaque_keys.edx.keys import CourseKey
 from rest_framework import status
 
 from analytics_data_api.utils import get_filename_safe_course_id
+from analytics_data_api.v0.tests.utils import flatten
+
 
 DEMO_COURSE_ID = u'course-v1:edX+DemoX+Demo_2014'
 SANITIZED_DEMO_COURSE_ID = get_filename_safe_course_id(DEMO_COURSE_ID)
@@ -39,3 +43,48 @@ def verify_bad_course_id(self, response, course_id='malformed-course-id'):
             u"developer_message": u"Course id/key {} malformed.".format(course_id)
         }
         self.assertDictEqual(json.loads(response.content), expected)
+
+
+class VerifyCsvResponseMixin(object):
+
+    def assertCsvResponseIsValid(self, response, expected_filename, expected_data=None, expected_headers=None):
+
+        # Validate the basic response status, content type, and filename
+        self.assertEquals(response.status_code, 200)
+        if expected_data:
+            self.assertEquals(response['Content-Type'].split(';')[0], 'text/csv')
+        self.assertEquals(response['Content-Disposition'], u'attachment; filename={}'.format(expected_filename))
+
+        # Validate other response headers
+        if expected_headers:
+            for header_name, header_content in expected_headers.iteritems():
+                self.assertEquals(response.get(header_name), header_content)
+
+        # Validate the content data
+        if expected_data:
+            data = map(flatten, expected_data)
+
+            # The CSV renderer sorts the headers alphabetically
+            fieldnames = sorted(data[0].keys())
+
+            # Generate the expected CSV output
+            expected = StringIO.StringIO()
+            writer = csv.DictWriter(expected, fieldnames)
+            writer.writeheader()
+            writer.writerows(data)
+            self.assertEqual(response.content, expected.getvalue())
+        else:
+            self.assertEqual(response.content, '')
+
+    def assertResponseFields(self, response, fields):
+        content_type = response.get('Content-Type', '').split(';')[0]
+        self.assertEquals(content_type, 'text/csv')
+
+        data = StringIO.StringIO(response.content)
+        reader = csv.reader(data)
+        rows = []
+        for row in reader:
+            rows.append(row)
+        # Just check the header row
+        self.assertGreater(len(rows), 1)
+        self.assertEqual(rows[0], fields)

analytics_data_api/v0/serializers.py

@@ -3,8 +3,6 @@
 # change for versions greater than 1.0.0. Tests target a specific version of the API, additional tests should be added
 # for subsequent versions if there are breaking changes introduced in those versions.
 
-import StringIO
-import csv
 import datetime
 from itertools import groupby
 import urllib
@@ -18,8 +16,9 @@
 from analytics_data_api.v0 import models
 from analytics_data_api.constants import country, enrollment_modes, genders
 from analytics_data_api.v0.models import CourseActivityWeekly
-from analytics_data_api.v0.tests.utils import flatten
-from analytics_data_api.v0.tests.views import DemoCourseMixin, DEMO_COURSE_ID, SANITIZED_DEMO_COURSE_ID
+from analytics_data_api.v0.tests.views import (
+    DemoCourseMixin, VerifyCsvResponseMixin, DEMO_COURSE_ID, SANITIZED_DEMO_COURSE_ID,
+)
 from analyticsdataserver.tests import TestCaseWithAuthentication
 
 
@@ -38,7 +37,7 @@ def test_default_fill(self):
 
 
 # pylint: disable=no-member
-class CourseViewTestCaseMixin(DemoCourseMixin):
+class CourseViewTestCaseMixin(DemoCourseMixin, VerifyCsvResponseMixin):
     model = None
     api_root_path = '/api/v0/'
     path = None
@@ -66,7 +65,8 @@ def get_latest_data(self, course_id=None):
         """
         raise NotImplementedError
 
-    def get_csv_filename(self):
+    @property
+    def csv_filename(self):
         return u'edX-DemoX-Demo_2014--{0}.csv'.format(self.csv_filename_slug)
 
     def test_get_not_found(self):
@@ -93,28 +93,12 @@ def assertCSVIsValid(self, course_id, filename):
         csv_content_type = 'text/csv'
         response = self.authenticated_get(path, HTTP_ACCEPT=csv_content_type)
 
-        # Validate the basic response status, content type, and filename
-        self.assertEquals(response.status_code, 200)
-        self.assertEquals(response['Content-Type'].split(';')[0], csv_content_type)
-        self.assertEquals(response['Content-Disposition'], u'attachment; filename={}'.format(filename))
-
-        # Validate the actual data
         data = self.format_as_response(*self.get_latest_data(course_id=course_id))
-        data = map(flatten, data)
-
-        # The CSV renderer sorts the headers alphabetically
-        fieldnames = sorted(data[0].keys())
-
-        # Generate the expected CSV output
-        expected = StringIO.StringIO()
-        writer = csv.DictWriter(expected, fieldnames)
-        writer.writeheader()
-        writer.writerows(data)
-        self.assertEqual(response.content, expected.getvalue())
+        self.assertCsvResponseIsValid(response, filename, data)
 
     def test_get_csv(self):
         """ Verify the endpoint returns data that has been properly converted to CSV. """
-        self.assertCSVIsValid(self.course_id, self.get_csv_filename())
+        self.assertCSVIsValid(self.course_id, self.csv_filename)
 
     def test_get_csv_with_deprecated_key(self):
         """