Merge pull request #488 from edx/alangsto/remove_custom_pagination

Fix: return first 10000 matches from ES search

Author: alangsto

analytics_data_api/v0/documents.py

@@ -160,7 +160,14 @@ def get_users_in_course(
             }
             for sort_policy in sort_policies
         ])
-        return search_request.execute()
+
+        # by default, ES will only return the first 10 documents, we want to get as many as possible
+        count = search_request.count()
+        if count > 10000:
+            logger.info('Cannot return more than 10,000 learners for course_id=%s', course_id)
+            count = 10000
+        total_search_requests = search_request[0:count]
+        return total_search_requests.execute()
 
     @classmethod
     def get_course_metadata(cls, course_id):

analytics_data_api/v0/tests/views/test_learners.py

@@ -441,34 +441,40 @@ def test_sort(self, learners, order_by, sort_order, expected_users):
         self.assert_learners_returned(response, expected_users)
 
     def test_pagination(self):
-        usernames = ['a', 'b', 'c', 'd', 'e']
-        self.create_learners([{'username': username, 'course_id': self.course_id} for username in usernames])
+        num_users = 16
+        learner_list = [{'username': 'test' + str(num), 'course_id': self.course_id} for num in range(num_users)]
+        self.create_learners(learner_list)
 
-        response = self._get(self.course_id, page_size=2)
+        response = self._get(self.course_id, page_size=5)
         payload = json.loads(response.content.decode('utf-8'))
         self.assertDictContainsSubset(
             {
-                'count': len(usernames),
+                'count': num_users,
                 'previous': None,
-                'next': self.expected_page_url(self.course_id, page=2, page_size=2),
-                'num_pages': 3
+                'next': self.expected_page_url(self.course_id, page=2, page_size=5),
+                'num_pages': 4
             },
             payload
         )
-        self.assert_learners_returned(response, [{'username': 'a'}, {'username': 'b'}])
+        self.assertEqual(response.status_code, 200)
+        returned_learners = json.loads(response.content.decode('utf-8'))['results']
+        self.assertEqual(len(returned_learners), 5)
 
-        response = self._get(self.course_id, page_size=2, page=3)
+        response = self._get(self.course_id, page_size=5, page=4)
         payload = json.loads(response.content.decode('utf-8'))
         self.assertDictContainsSubset(
             {
-                'count': len(usernames),
-                'previous': self.expected_page_url(self.course_id, page=2, page_size=2),
+                'count': num_users,
+                'previous': self.expected_page_url(self.course_id, page=3, page_size=5),
                 'next': None,
-                'num_pages': 3
+                'num_pages': 4
             },
             payload
         )
-        self.assert_learners_returned(response, [{'username': 'e'}])
+
+        self.assertEqual(response.status_code, 200)
+        returned_learners = json.loads(response.content.decode('utf-8'))['results']
+        self.assertEqual(len(returned_learners), 1)
 
     # Error cases
     @ddt.data(