build: add analytics api to devstack (#495)

Some settings to inherit env vars from devstack. Running make commands with tox is now optional.

Author: zacharis278

.travis.yml

@@ -12,16 +12,13 @@ install:
 - pip install -r requirements/travis.txt
 script:
 - docker exec -t -e TRAVIS=1 analytics_api_testing bash -c " cd /edx/app/analytics_api/analytics_api/
-  && make $TARGETS "
+  && export TOXENV=django32 && make test.requirements tox.requirements $TARGETS "
 matrix:
   include:
   - python: '3.8'
-    env: TESTNAME=quality-python-3.8 TARGETS="PYTHON_ENV=py38 quality"
+    env: TESTNAME=quality-python-3.8 TARGETS="quality"
   - python: '3.8'
-    env: TESTNAME=test-python-3.8 TARGETS="PYTHON_ENV=py38 main.test"
-  - python: '3.8'
-    env: TESTNAME=test-python-3.8-django-3.0 TARGETS="PYTHON_ENV=py38 DJANGO_VERSION=django32
-      main.test"
+    env: TESTNAME=test-python-3.8 TARGETS="main.test"
     after_success:
     - docker exec analytics_api_testing /edx/app/analytics_api/analytics_api/.travis/run_coverage.sh
     - codecov --disable pycov

Makefile

@@ -1,11 +1,14 @@
 ROOT = $(shell echo "$$PWD")
 COVERAGE_DIR = $(ROOT)/build/coverage
-PACKAGES = analyticsdataserver analytics_data_api
 DATABASES = default analytics
-PYTHON_ENV=py38
-DJANGO_VERSION=django32
 .DEFAULT_GOAL := help
 
+TOX=''
+
+ifdef TOXENV
+TOX := tox -- #to isolate each tox environment if TOXENV is defined
+endif
+
 help: ## display this help message
 	@echo "Please use \`make <target>' where <target> is one of"
 	@perl -nle'print $& if m{^[\.a-zA-Z_-]+:.*?## .*$$}' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m  %-25s\033[0m %s\n", $$1, $$2}'
@@ -28,7 +31,7 @@ test.requirements: requirements  ## install base and test requirements
 	pip3 install -q -r requirements/test.txt
 
 tox.requirements:  ## install tox requirements
-	 pip3 install -q -r requirements/tox.txt
+	pip3 install -q -r requirements/tox.txt
 
 develop: test.requirements  ## install test and dev requirements
 	pip3 install -q -r requirements/dev.txt
@@ -58,16 +61,21 @@ upgrade:
 	mv requirements/test.tmp requirements/test.txt
 
 
-clean: tox.requirements  ## install tox requirements and run tox clean. Delete *.pyc files
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-clean
+clean:
+	$(TOX)coverage erase
 	find . -name '*.pyc' -delete
 
-main.test: tox.requirements clean
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-tests
+main.test: clean
 	export COVERAGE_DIR=$(COVERAGE_DIR) && \
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-coverage
+	$(TOX)pytest --cov-report html --cov-report xml
 
+test:
+
+ifeq ($(DJANGO_SETTINGS_MODULE),analyticsdataserver.settings.devstack)
+test: main.test
+else
 test: test.run_elasticsearch main.test test.stop_elasticsearch
+endif
 
 diff.report: test.requirements  ## Show the diff in quality and coverage
 	diff-cover $(COVERAGE_DIR)/coverage.xml --html-report $(COVERAGE_DIR)/diff_cover.html
@@ -79,19 +87,19 @@ view.diff.report:  ## Show the diff in quality and coverage using xdg
 	xdg-open file:///$(COVERAGE_DIR)/diff_quality_pycodestyle.html
 	xdg-open file:///$(COVERAGE_DIR)/diff_quality_pylint.html
 
-run_check_isort: tox.requirements  ## Run tox check_isort. (Installs tox requirements.)
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-check_isort
+run_check_isort:
+	$(TOX)isort --check-only --recursive --diff analytics_data_api/ analyticsdataserver/
 
-run_pycodestyle: tox.requirements  ## Run tox pycodestyle. (Installs tox requirements.)
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-pycodestyle
+run_pycodestyle:
+	$(TOX)pycodestyle --config=.pycodestyle analytics_data_api analyticsdataserver
 
-run_pylint: tox.requirements  ## Run tox pylint. (Installs tox requirements.)
-	tox -e $(PYTHON_ENV)-$(DJANGO_VERSION)-pylint
+run_pylint:
+	$(TOX)pylint -j 0 --rcfile=pylintrc analytics_data_api analyticsdataserver
 
-run_isort: tox.requirements  ## Run tox isort. (Installs tox requirements.)
-	tox -e  $(PYTHON_ENV)-$(DJANGO_VERSION)-isort
+run_isort:
+	$(TOX)isort --recursive analytics_data_api/ analyticsdataserver/
 
-quality: tox.requirements run_pylint run_check_isort run_pycodestyle  ## run_pylint, run_check_isort, run_pycodestyle (Installs tox requirements.)
+quality: run_pylint run_check_isort run_pycodestyle  ## run_pylint, run_check_isort, run_pycodestyle (Installs tox requirements.)
 
 validate: test.requirements test quality  ## Runs make test and make quality. (Installs test requirements.)
 
@@ -108,6 +116,9 @@ loaddata: migrate  ## Runs migrations and generates fake data
 	python manage.py loaddata problem_response_answer_distribution --database=analytics
 	python manage.py generate_fake_course_data
 
+create_indices:  ## Create ElasticSearch indices
+	python manage.py create_elasticsearch_learners_indices
+
 demo: clean requirements loaddata  ## Runs make clean, requirements, and loaddata, sets api key to edx
 	python manage.py set_api_key edx edx
 
@@ -134,6 +145,5 @@ travis_docker_push: travis_docker_tag travis_docker_auth ## push to docker hub
 	docker push 'openedx/analytics-data-api:latest-newrelic'
 	docker push "openedx/analytics-data-api:$$TRAVIS_COMMIT-newrelic"
 
-docs:
-	pip install -r requirements/tox.txt
+docs: tox.requirements
 	tox -e docs

README.rst

@@ -15,8 +15,15 @@ unless otherwise noted.
 
 Please see ``LICENSE.txt`` for details.
 
-Getting Started
----------------
+Getting Started With Devstack
+-----------------------------
+The best way to run this service is with edX Devstack: https://github.com/edx/devstack.
+
+See the [Devstack README](https://github.com/edx/devstack/blob/master/README.rst) for information on how to install and run the analytics data api.
+
+
+Getting Started Standalone
+--------------------------
 #. Create a virtual environment and activate it.
 
 #. Install the requirements:
@@ -74,6 +81,15 @@ Development with edx-enterprise-data
 If you need to make changes to ``edx-enterprise-data`` and have them reflected when you run the ``edx-analytics-data-api`` server,
 you can follow these steps. If you do not intend to make changes to ``edx-enterprise-data``, you can skip this section.
 
+Devstack Development
+~~~~~~~~~~~~~~~~~~~~
+#. Clone the `edx-enterprise-data <https://github.com/edx/edx-enterprise-data>`_ repo into the ``src`` folder alongside your devstack 
+workspace on the host system. 
+#. The modified ``edx-enterprise-data`` repository will now be available to devstack containers at ``/edx/src/edx-enterprise-data``.
+#. Inside the ``edx-data-analytics-api`` container run `pip install `-e /edx/src/edx-enterprise-data``
+
+Standalone Development
+~~~~~~~~~~~~~~~~~~~~~~
 #. Recommended: Install this repo into a subfolder of your working directory. Within that subfolder create an ``src`` folder.
 #. Clone the `edx-enterprise-data <https://github.com/edx/edx-enterprise-data>`_ repo into the ``src`` folder.
 #. ``cd`` into your ``edx-data-analytics-api`` folder and activate your virtualenv.

analyticsdataserver/settings/devstack.py

@@ -1,9 +1,10 @@
 """Devstack settings."""
 
-from analyticsdataserver.settings.local import *
+import os
 
-ALLOWED_HOSTS += ['edx.devstack.analyticsapi']
+from analyticsdataserver.settings.local import *
 
+########## DATABASE CONFIGURATION
 DATABASES = {
     'default': {
         'ENGINE': 'django.db.backends.mysql',
@@ -23,5 +24,20 @@
     }
 }
 
-ELASTICSEARCH_LEARNERS_HOST = "edx.devstack.elasticsearch"
+DB_OVERRIDES = dict(
+    USER=os.environ.get('DB_USER', DATABASES['default']['USER']),
+    PASSWORD=os.environ.get('DB_PASSWORD', DATABASES['default']['PASSWORD']),
+    HOST=os.environ.get('DB_HOST', DATABASES['default']['HOST']),
+    PORT=os.environ.get('DB_PORT', DATABASES['default']['PORT']),
+)
+
+for override, value in DB_OVERRIDES.items():
+    DATABASES['default'][override] = value
+    DATABASES['analytics'][override] = value
+########## END DATABASE CONFIGURATION
+
+ELASTICSEARCH_LEARNERS_HOST = os.environ.get('ELASTICSEARCH_LEARNERS_HOST', 'edx.devstack.elasticsearch')
+
+ALLOWED_HOSTS += ['edx.devstack.analyticsapi']
+
 LMS_BASE_URL = "http://edx.devstack.lms:18000/"

pytest.ini

@@ -1,3 +1,2 @@
 [pytest]
-DJANGO_SETTINGS_MODULE = analyticsdataserver.settings.test
-addopts = --cov analytics_data_api --cov-report term-missing --cov-config=.coveragerc --no-cov-on-fail -p no:randomly
+addopts = --ds=analyticsdataserver.settings.test --cov analytics_data_api --cov-report term-missing --cov-config=.coveragerc --no-cov-on-fail -p no:randomly

requirements/pip_tools.txt

@@ -8,7 +8,7 @@ click==8.0.1
     # via pip-tools
 pep517==0.11.0
     # via pip-tools
-pip-tools==6.3.0
+pip-tools==6.4.0
     # via -r requirements/pip_tools.in
 six==1.16.0
     # via -r requirements/pip_tools.in

tox.ini

@@ -1,6 +1,6 @@
 [tox]
 skipsdist = True
-envlist = py38-django{22,32}-{check_isort,pycodestyle,pylint,isort,tests}
+envlist = py38-django{22,32}
 
 [testenv]
 passenv =
@@ -15,14 +15,7 @@ deps =
 	django32: Django>=3.2,<3.3
 	-r requirements/test.txt
 commands =
-	check_isort: isort --check-only --recursive --diff analytics_data_api/ analyticsdataserver/
-	isort: isort --recursive analytics_data_api/ analyticsdataserver/
-	clean: coverage erase
-	pycodestyle: pycodestyle --config=.pycodestyle analytics_data_api analyticsdataserver
-	pylint: pylint -j 0 --rcfile=pylintrc analytics_data_api analyticsdataserver
-	tests: python -Wd -m pytest {posargs}
-	coverage: coverage html
-	coverage: coverage xml
+	{posargs:pytest}
 
 [testenv:docs]
 deps =