diff --git a/.travis.yml b/.travis.yml index 405a2da6..e0f1dbb7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,3 +15,8 @@ script: - make validate after_success: - codecov +deploy: + provider: script + script: make docker_push + on: + branch: master diff --git a/.tx/config b/.tx/config index 8a27a240..df58914b 100644 --- a/.tx/config +++ b/.tx/config @@ -1,14 +1,14 @@ [main] host = https://www.transifex.com -[edx-platform.license_manager] -file_filter = license_manager/conf/locale//LC_MESSAGES/django.po -source_file = license_manager/conf/locale/en/LC_MESSAGES/django.po +[edx-platform.license-manager] +file_filter = license-manager/conf/locale//LC_MESSAGES/django.po +source_file = license-manager/conf/locale/en/LC_MESSAGES/django.po source_lang = en type = PO -[edx-platform.license_manager-js] -file_filter = license_manager/conf/locale//LC_MESSAGES/djangojs.po -source_file = license_manager/conf/locale/en/LC_MESSAGES/djangojs.po +[edx-platform.license-manager-js] +file_filter = license-manager/conf/locale//LC_MESSAGES/djangojs.po +source_file = license-manager/conf/locale/en/LC_MESSAGES/djangojs.po source_lang = en type = PO diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..5e7323a5 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,64 @@ +FROM ubuntu:bionic as app +MAINTAINER devops@edx.org + + +# Packages installed: +# git; Used to pull in particular requirements from github rather than pypi, +# and to check the sha of the code checkout. + +# language-pack-en locales; ubuntu locale support so that system utilities have a consistent +# language and time zone. + +# python; ubuntu doesnt ship with python, so this is the python we will use to run the application + +# python3-pip; install pip to install application requirements.txt files + +# libssl-dev; # mysqlclient wont install without this. + +# libmysqlclient-dev; to install header files needed to use native C implementation for +# MySQL-python for performance gains. + +# If you add a package here please include a comment above describing what it is used for +RUN apt-get update && apt-get upgrade -qy && apt-get install language-pack-en locales git python3.5 python3-pip libmysqlclient-dev libssl-dev python3-dev -qy && \ +pip3 install --upgrade pip setuptools && \ +rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/bin/pip3 /usr/bin/pip +RUN ln -s /usr/bin/python3 /usr/bin/python + +RUN locale-gen en_US.UTF-8 +ENV LANG en_US.UTF-8 +ENV LANGUAGE en_US:en +ENV LC_ALL en_US.UTF-8 +ENV DJANGO_SETTINGS_MODULE license_manager.settings.production + +EXPOSE 18170 +RUN useradd -m --shell /bin/false app + +WORKDIR /edx/app/license_manager + +# Copy the requirements explicitly even though we copy everything below +# this prevents the image cache from busting unless the dependencies have changed. +COPY requirements/production.txt /edx/app/license_manager/requirements/production.txt + +# Dependencies are installed as root so they cannot be modified by the application user. +RUN pip3 install -r requirements/production.txt + +RUN mkdir -p /edx/var/log + +# Code is owned by root so it cannot be modified by the application user. +# So we copy it before changing users. +USER app + +# Gunicorn 19 does not log to stdout or stderr by default. Once we are past gunicorn 19, the logging to STDOUT need not be specified. +CMD gunicorn --workers=2 --name license_manager -c /edx/app/license_manager/license_manager/docker_gunicorn_configuration.py --log-file - --max-requests=1000 license_manager.wsgi:application + +# This line is after the requirements so that changes to the code will not +# bust the image cache +COPY . /edx/app/license_manager + + +FROM app as newrelic +RUN pip install newrelic +CMD newrelic-admin run-program gunicorn --workers=2 --name license_manager -c /edx/app/license_manager/license_manager/docker_gunicorn_configuration.py --log-file - --max-requests=1000 license_manager.wsgi:application + diff --git a/Makefile b/Makefile index 2abdb061..0d8842c0 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.DEFAULT_GOAL := test +.DEFAULT_GOAL := help .PHONY: help clean piptools requirements ci_requirements dev_requirements \ validation_requirements doc_requirementsprod_requirements static shell \ @@ -133,3 +133,20 @@ detect_changed_source_translations: ## check if translation files are up-to-date cd license_manager && i18n_tool changed validate_translations: fake_translations detect_changed_source_translations ## install fake translations and check if translation files are up-to-date + +docker_build: + docker build . -f Dockerfile -t openedx/license_manager + docker build . -f Dockerfile --target newrelic -t openedx/license_manager:latest-newrelic + +docker_tag: docker_build + docker tag openedx/license_manager openedx/license_manager:$$TRAVIS_COMMIT + docker tag openedx/license_manager:latest-newrelic openedx/license_manager:$$TRAVIS_COMMIT-newrelic + +docker_auth: + echo "$$DOCKER_PASSWORD" | docker login -u "$$DOCKER_USERNAME" --password-stdin + +docker_push: docker_tag docker_auth ## push to docker hub + docker push 'openedx/license_manager:latest' + docker push "openedx/license_manager:$$TRAVIS_COMMIT" + docker push 'openedx/license_manager:latest-newrelic' + docker push "openedx/license_manager:$$TRAVIS_COMMIT-newrelic" diff --git a/README.rst b/README.rst index dac63101..72439013 100644 --- a/README.rst +++ b/README.rst @@ -1,4 +1,4 @@ -License Management Service |Travis|_ |Codecov|_ +License Manager |Travis|_ |Codecov|_ =================================================== .. |Travis| image:: https://travis-ci.org/edx/license-manager.svg?branch=master .. _Travis: https://travis-ci.org/edx/license-manager diff --git a/docker-compose.yml b/docker-compose.yml index 9ad6075e..5535b69a 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,13 +13,11 @@ license_manager: # Uncomment this line to use the official license_manager base image image: license_manager:v1 - # Uncomment the next two lines to build from a local configuration repo - #build: ../configuration - #dockerfile: docker/build/license_manager/Dockerfile - container_name: license_manager volumes: - .:/edx/app/license_manager/license_manager - command: /edx/app/license_manager/devstack.sh start + command: bash -c 'gunicorn --reload --workers=2 --name license_manager -b :18170 -c /edx/app/license_manager/license_manager/docker_gunicorn_configuration.py --log-file - --max-requests=1000 license_manager.wsgi:application' + environment: + DJANGO_SETTINGS_MODULE: license_manager.settings.devstack ports: - - "18170:18170" + - "18170:18170" # TODO: change this to your port diff --git a/docs/conf.py b/docs/conf.py index 79f67f4e..f72169d8 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # -# License Management Service documentation build configuration file, created by +# License Manager documentation build configuration file, created by # sphinx-quickstart on Sun Feb 17 11:46:20 2013. # # This file is execfile()d with the current directory set to its containing dir. @@ -48,7 +48,7 @@ master_doc = 'index' # General information about the project. -project = u'License Management Service' +project = u'License Manager' copyright = edx_theme.COPYRIGHT author = u'edX' @@ -192,7 +192,7 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'license_manager.tex', u'License Management Service Documentation', + ('index', 'license_manager.tex', u'License Manager Documentation', u'edX', 'manual'), ] @@ -222,7 +222,7 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'License Management Service', u'License Management Service Documentation', + ('index', 'License Manager', u'License Manager Documentation', [u'edX'], 1) ] @@ -236,8 +236,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'License Management Service', u'License Management Service Documentation', - u'edX', 'License Management Service', 'License Management Service', + ('index', 'License Manager', u'License Manager Documentation', + u'edX', 'License Manager', 'License Manager', 'Miscellaneous' ), ] diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 4ca9e6f0..4c68d3ef 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -7,65 +7,39 @@ below is executed within the virtualenv. .. _virtualenv: https://virtualenvwrapper.readthedocs.org/en/latest/ -Initialize and Provision ------------------------- - 1. Start and provision the edX `devstack `_, as license-manager currently relies on devstack - 2. Verify that your virtual environment is active before proceeding - 3. Clone the license-manager repo and cd into that directory - 4. Run *make dev.provision* to provision a new license manager environment - 5. Run *make dev.init* to start the license manager app and run migrations - -Viewing License Manager ------------------------- -Once the server is up and running you can view the license manager at http://localhost:18170/admin. - -You can login with the username *edx@example.com* and password *edx*. - -Makefile Commands +The following is provided for informational purposes only. You can likely ignore this section. +======= +Install dependencies -------------------- -The `Makefile <../Makefile>`_ includes numerous commands to start the service, but the basic commands are the following: - -Start the Docker containers to run the license manager servers - -.. code-block:: bash - - $ make dev.up - -Open the shell to the license manager container for manual commands - -.. code-block:: bash - - $ make app-shell - -Open the logs in the license manager container +Dependencies can be installed via the command below. .. code-block:: bash - $ make license-manager-logs + $ make requirements -Advanced Setup Outside Docker -============================= -The following is provided for informational purposes only. You can likely ignore this section. Local/Private Settings ---------------------- When developing locally, it may be useful to have settings overrides that you do not wish to commit to the repository. -If you need such overrides, create a file :file:`license_manager/settings/private.py`. This file's values are -read by :file:`license_manager/settings/local.py`, but ignored by Git. +If you need such overrides, create a file :file:`license-manager/settings/private.py`. This file's values are +read by :file:`license-manager/settings/local.py`, but ignored by Git. + +Configure edX OAuth (Optional) +------------------------------- + +OAuth only needs to be configured if the IDA would like to use the LMS's authentication functionality in place of managing its own. -Configure edX OAuth -------------------- -This service relies on the LMS server as the OAuth 2.0 authentication provider. +This functionality relies on the LMS server as the OAuth 2.0 authentication provider. -Configuring License Manager service to communicate with other IDAs using OAuth requires registering a new client with the authentication +Configuring License Manager to communicate with other IDAs using OAuth requires registering a new client with the authentication provider (LMS) and updating the Django settings for this project with the generated client credentials. A new OAuth 2.0 client can be created when using Devstack by visiting ``http://127.0.0.1:18000/admin/oauth2_provider/application/``. 1. Click the :guilabel:`Add Application` button. 2. Leave the user field blank. - 3. Specify the name of this service, ``License Manager service``, as the client name. - 4. Set the :guilabel:`URL` to the root path of this service: ``http://127.0.0.1:8003/``. - 5. Set the :guilabel:`Redirect URL` to the complete endpoint: ``http://127.0.0.1:18150/complete/edx-oauth2/``. + 3. Specify the name of this service, ``License Manager``, as the client name. + 4. Set the :guilabel:`URL` to the root path of this service: ``http://127.0.0.1:18170/``. + 5. Set the :guilabel:`Redirect URL` to the complete endpoint: ``http://127.0.0.1:18170/complete/edx-oauth2/``. 6. Copy the :guilabel:`Client ID` and :guilabel:`Client Secret` values. They will be used later. 7. Select :guilabel:`Confidential` as the client type. 8. Select :guilabel:`Authorization code` as the authorization grant type. @@ -74,7 +48,7 @@ A new OAuth 2.0 client can be created when using Devstack by visiting ``http://1 Now that you have the client credentials, you can update your settings (ideally in -:file:`license_manager/settings/local.py`). The table below describes the relevant settings. +:file:`license-manager/settings/local.py`). The table below describes the relevant settings. +-----------------------------------+----------------------------------+--------------------------------------------------------------------------+ | Setting | Description | Value | diff --git a/docs/index.rst b/docs/index.rst index b00830a4..d829c77f 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,9 +1,9 @@ -.. license-manager documentation master file, created by +.. license_manager documentation master file, created by sphinx-quickstart on Sun Feb 17 11:46:20 2013. You can adapt this file completely to your liking, but it should at least contain the root `toctree` directive. -License Management Service +License Manager ======================================================================= Django backend for managing licenses and subscriptions diff --git a/license_manager/docker_gunicorn_configuration.py b/license_manager/docker_gunicorn_configuration.py new file mode 100644 index 00000000..6449c310 --- /dev/null +++ b/license_manager/docker_gunicorn_configuration.py @@ -0,0 +1,56 @@ +""" +gunicorn configuration file: http://docs.gunicorn.org/en/develop/configure.html +""" +import multiprocessing # pylint: disable=unused-import + + +preload_app = True +timeout = 300 +bind = "0.0.0.0:18170" + +workers = 2 + + +def pre_request(worker, req): + worker.log.info("%s %s" % (req.method, req.path)) + + +def close_all_caches(): + """ + Close the cache so that newly forked workers cannot accidentally share + the socket with the processes they were forked from. This prevents a race + condition in which one worker could get a cache response intended for + another worker. + """ + # We do this in a way that is safe for 1.4 and 1.8 while we still have some + # 1.4 installations. + from django.conf import settings + from django.core import cache as django_cache + if hasattr(django_cache, 'caches'): + get_cache = django_cache.caches.__getitem__ + else: + get_cache = django_cache.get_cache # pylint: disable=no-member + for cache_name in settings.CACHES: + cache = get_cache(cache_name) + if hasattr(cache, 'close'): + cache.close() + + # The 1.4 global default cache object needs to be closed also: 1.4 + # doesn't ensure you get the same object when requesting the same + # cache. The global default is a separate Python object from the cache + # you get with get_cache("default"), so it will have its own connection + # that needs to be closed. + cache = django_cache.cache + if hasattr(cache, 'close'): + cache.close() + + +def post_fork(server, worker): # pylint: disable=unused-argument + close_all_caches() + +def when_ready(server): # pylint: disable=unused-argument + """When running in debug mode, run Django's `check` to better match what `manage.py runserver` does""" + from django.conf import settings + from django.core.management import call_command + if settings.DEBUG: + call_command("check") diff --git a/license_manager/settings/base.py b/license_manager/settings/base.py index a1ea7f97..2090c267 100644 --- a/license_manager/settings/base.py +++ b/license_manager/settings/base.py @@ -3,6 +3,8 @@ from corsheaders.defaults import default_headers as corsheaders_default_headers +from license_manager.settings.utils import get_logger_config + # PATH vars here = lambda *x: join(abspath(dirname(__file__)), *x) PROJECT_ROOT = here("..") @@ -57,6 +59,7 @@ 'edx_rest_framework_extensions.auth.jwt.middleware.JwtAuthCookieMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'django.contrib.auth.middleware.SessionAuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', 'social_django.middleware.SocialAuthExceptionMiddleware', @@ -223,48 +226,4 @@ # END OPENEDX-SPECIFIC CONFIGURATION # Set up logging for development use (logging to stdout) -LOGGING = { - 'version': 1, - 'disable_existing_loggers': False, - 'formatters': { - 'standard': { - 'format': '%(asctime)s %(levelname)s %(process)d ' - '[%(name)s] %(filename)s:%(lineno)d - %(message)s', - }, - }, - 'handlers': { - 'console': { - 'level': 'INFO', - 'class': 'logging.StreamHandler', - 'formatter': 'standard', - 'stream': 'ext://sys.stdout', - }, - }, - 'loggers': { - 'django': { - 'handlers': ['console'], - 'propagate': True, - 'level': 'INFO' - }, - 'requests': { - 'handlers': ['console'], - 'propagate': True, - 'level': 'WARNING' - }, - 'factory': { - 'handlers': ['console'], - 'propagate': True, - 'level': 'WARNING' - }, - 'django.request': { - 'handlers': ['console'], - 'propagate': True, - 'level': 'WARNING' - }, - '': { - 'handlers': ['console'], - 'level': 'DEBUG', - 'propagate': False - }, - } -} +LOGGING = get_logger_config(debug=DEBUG, dev_env=True, local_loglevel='DEBUG') diff --git a/license_manager/settings/local.py b/license_manager/settings/local.py index 747e7af8..6db1e45b 100644 --- a/license_manager/settings/local.py +++ b/license_manager/settings/local.py @@ -62,6 +62,8 @@ ENABLE_AUTO_AUTH = True +LOGGING = get_logger_config(debug=DEBUG, dev_env=True, local_loglevel='DEBUG') + ##################################################################### # Lastly, see if the developer has any local overrides. if os.path.isfile(join(dirname(abspath(__file__)), 'private.py')): diff --git a/license_manager/settings/utils.py b/license_manager/settings/utils.py index 0065bf01..6a71c311 100644 --- a/license_manager/settings/utils.py +++ b/license_manager/settings/utils.py @@ -1,4 +1,7 @@ -from os import environ +import platform +import sys +from logging.handlers import SysLogHandler +from os import environ, path from django.core.exceptions import ImproperlyConfigured @@ -10,3 +13,112 @@ def get_env_setting(setting): except KeyError: error_msg = "Set the [%s] env variable!" % setting raise ImproperlyConfigured(error_msg) + +def get_logger_config(log_dir='/var/tmp', + logging_env="no_env", + edx_filename="edx.log", + dev_env=False, + debug=False, + local_loglevel='INFO', + service_variant='license_manager'): + """ + Return the appropriate logging config dictionary. You should assign the + result of this to the LOGGING var in your settings. + If dev_env is set to true logging will not be done via local rsyslogd, + instead, application logs will be dropped in log_dir. + "edx_filename" is ignored unless dev_env is set to true since otherwise logging is handled by rsyslogd. + """ + + # Revert to INFO if an invalid string is passed in + if local_loglevel not in ['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL']: + local_loglevel = 'INFO' + + hostname = platform.node().split(".")[0] + syslog_format = ( + "[service_variant={service_variant}]" + "[%(name)s][env:{logging_env}] %(levelname)s " + "[{hostname} %(process)d] [%(filename)s:%(lineno)d] " + "- %(message)s" + ).format( + service_variant=service_variant, + logging_env=logging_env, hostname=hostname + ) + + if debug: + handlers = ['console'] + else: + handlers = ['local'] + + logger_config = { + 'version': 1, + 'disable_existing_loggers': False, + 'formatters': { + 'standard': { + 'format': '%(asctime)s %(levelname)s %(process)d ' + '[%(name)s] %(filename)s:%(lineno)d - %(message)s', + }, + 'syslog_format': {'format': syslog_format}, + 'raw': {'format': '%(message)s'}, + }, + 'handlers': { + 'console': { + 'level': 'DEBUG' if debug else 'INFO', + 'class': 'logging.StreamHandler', + 'formatter': 'standard', + 'stream': sys.stdout, + }, + }, + 'loggers': { + 'django': { + 'handlers': handlers, + 'propagate': True, + 'level': 'INFO' + }, + 'requests': { + 'handlers': handlers, + 'propagate': True, + 'level': 'WARNING' + }, + 'factory': { + 'handlers': handlers, + 'propagate': True, + 'level': 'WARNING' + }, + 'django.request': { + 'handlers': handlers, + 'propagate': True, + 'level': 'WARNING' + }, + '': { + 'handlers': handlers, + 'level': 'DEBUG', + 'propagate': False + }, + } + } + + if dev_env: + edx_file_loc = path.join(log_dir, edx_filename) + logger_config['handlers'].update({ + 'local': { + 'class': 'logging.handlers.RotatingFileHandler', + 'level': local_loglevel, + 'formatter': 'standard', + 'filename': edx_file_loc, + 'maxBytes': 1024 * 1024 * 2, + 'backupCount': 5, + }, + }) + else: + logger_config['handlers'].update({ + 'local': { + 'level': local_loglevel, + 'class': 'logging.handlers.SysLogHandler', + # Use a different address for Mac OS X + 'address': '/var/run/syslog' if sys.platform == "darwin" else '/dev/log', + 'formatter': 'syslog_format', + 'facility': SysLogHandler.LOG_LOCAL0, + }, + }) + + return logger_config diff --git a/license_manager/wsgi.py b/license_manager/wsgi.py index f2219390..25964851 100644 --- a/license_manager/wsgi.py +++ b/license_manager/wsgi.py @@ -10,13 +10,19 @@ from os.path import abspath, dirname from sys import path +from django.conf import settings +from django.contrib.staticfiles.handlers import StaticFilesHandler from django.core.wsgi import get_wsgi_application SITE_ROOT = dirname(dirname(abspath(__file__))) path.append(SITE_ROOT) -os.environ.setdefault("DJANGO_SETTINGS_MODULE", "license_manager.settings.local") - - application = get_wsgi_application() # pylint: disable=invalid-name + +# Allows the gunicorn app to serve static files in development environment. +# Without this, css in django admin will not be served locally. +if settings.DEBUG: + application = StaticFilesHandler(get_wsgi_application()) +else: + application = get_wsgi_application() diff --git a/requirements/pip-tools.txt b/requirements/pip-tools.txt index 9e9b1940..c447b525 100644 --- a/requirements/pip-tools.txt +++ b/requirements/pip-tools.txt @@ -4,6 +4,6 @@ # # make upgrade # -click==7.1.1 # via pip-tools -pip-tools==4.5.1 -six==1.14.0 # via pip-tools +click==7.0 # via pip-tools +pip-tools==4.4.1 +six==1.12.0 # via pip-tools