Add health_check framework

Add the possibility to add a prefix to all the URLs managed by c2cwsgiutils.
Just define `c2c.base_path` in the production.ini or the `C2C_BASE_PATH`
environment variable.

.gitignore

@@ -1,6 +1,7 @@
 /.venv/
 /acceptance_tests/app/c2cwsgiutils
 /acceptance_tests/app/c2cwsgiutils_run
+/acceptance_tests/app/c2cwsgiutils_genversion.py
 /acceptance_tests/app/rel_requirements.txt
 /acceptance_tests/app/setup.cfg
 /acceptance_tests/tests/c2cwsgiutils

README.md

@@ -14,6 +14,7 @@ applications:
   SQL_PROFILER_SECRET env var and using the view (/sql_profiler) to switch it ON and OFF. Warning,
   it will slow down everything.
 * A view to get the version information about the application and the installed packages (/versions.json)
+* A framework for implementing a health_check service (/health_check)
 * Error handlers to send JSON messages to the client in case of error
 * A cornice service drop in replacement for setting up CORS
 

acceptance_tests/app/c2cwsgiutils_app/__init__.py

@@ -1,15 +1,26 @@
+
 import c2cwsgiutils.pyramid
+from c2cwsgiutils.health_check import HealthCheck
 from pyramid.config import Configurator
+from pyramid.httpexceptions import HTTPInternalServerError
 
 from c2cwsgiutils_app import models
 
 
+def _failure(_request):
+    raise HTTPInternalServerError('failing check')
+
+
 def main(_, **settings):
     """ This function returns a Pyramid WSGI application.
     """
     config = Configurator(settings=settings, route_prefix='/api')
     config.include(c2cwsgiutils.pyramid.includeme)
     models.init(config)
     config.scan("c2cwsgiutils_app.services")
+    health_check = HealthCheck(config)
+    health_check.add_db_session_check(models.DBSession, at_least_one_model=models.Hello)
+    health_check.add_url_check('http://localhost/api/hello')
+    health_check.add_custom_check('fail', _failure, 2)
 
     return config.make_wsgi_app()

acceptance_tests/app/production.ini

@@ -22,6 +22,8 @@ sqlalchemy_slave.pool_recycle = 30
 sqlalchemy_slave.pool_size = 5
 sqlalchemy_slave.max_overflow = 25
 
+c2c.base_path = /c2c
+
 ###
 # logging configuration
 # http://docs.pylonsproject.org/projects/pyramid/en/1.6-branch/narr/logging.html

acceptance_tests/tests/tests/test_health_check.py

@@ -0,0 +1,36 @@
+import json
+
+
+def test_ok(app_connection):
+    response = app_connection.get_json("c2c/health_check")
+    print('response=' + json.dumps(response))
+    assert response == {
+        'status': 200,
+        'successes': ['db_engine_sqlalchemy', 'db_engine_sqlalchemy_slave', 'http://localhost/api/hello'],
+        'failures': {}
+    }
+
+
+def test_failure(app_connection):
+    response = app_connection.get_json("c2c/health_check", params={'max_level': '2'}, expected_status=500)
+    print('response=' + json.dumps(response))
+    assert response == {
+        'status': 500,
+        'successes': ['db_engine_sqlalchemy', 'db_engine_sqlalchemy_slave', 'http://localhost/api/hello'],
+        'failures': {
+            'fail': {
+                'message': 'failing check',
+                'stacktrace': response['failures']['fail']['stacktrace']
+            }
+        }
+    }
+
+
+def test_ping(app_connection):
+    response = app_connection.get_json("c2c/health_check", params={'max_level': '0'})
+    print('response=' + json.dumps(response))
+    assert response == {
+        'status': 200,
+        'successes': [],
+        'failures': {}
+    }

acceptance_tests/tests/tests/test_logging.py

@@ -1,41 +1,41 @@
 def test_api(app_connection):
-    response = app_connection.get_json('logging/level',
+    response = app_connection.get_json('c2c/logging/level',
                                        params={'secret': 'changeme', 'name': 'sqlalchemy.engine'})
     assert response == {'status': 200, 'name': 'sqlalchemy.engine', 'level': 'DEBUG',
                         'effective_level': 'DEBUG'}
 
-    response = app_connection.get_json('logging/level',
+    response = app_connection.get_json('c2c/logging/level',
                                        params={'secret': 'changeme', 'name': 'sqlalchemy.engine.sub'})
     assert response == {'status': 200, 'name': 'sqlalchemy.engine.sub', 'level': 'NOTSET',
                         'effective_level': 'DEBUG'}
 
-    response = app_connection.get_json('logging/level',
+    response = app_connection.get_json('c2c/logging/level',
                                        params={'secret': 'changeme', 'name': 'sqlalchemy.engine',
                                                'level': 'INFO'})
     assert response == {'status': 200, 'name': 'sqlalchemy.engine', 'level': 'INFO',
                         'effective_level': 'INFO'}
 
-    response = app_connection.get_json('logging/level',
+    response = app_connection.get_json('c2c/logging/level',
                                        params={'secret': 'changeme', 'name': 'sqlalchemy.engine'})
     assert response == {'status': 200, 'name': 'sqlalchemy.engine', 'level': 'INFO',
                         'effective_level': 'INFO'}
 
-    response = app_connection.get_json('logging/level',
+    response = app_connection.get_json('c2c/logging/level',
                                        params={'secret': 'changeme', 'name': 'sqlalchemy.engine.sub'})
     assert response == {'status': 200, 'name': 'sqlalchemy.engine.sub', 'level': 'NOTSET',
                         'effective_level': 'INFO'}
 
-    response = app_connection.get_json('logging/level',
+    response = app_connection.get_json('c2c/logging/level',
                                        params={'secret': 'changeme', 'name': 'sqlalchemy.engine',
                                                'level': 'DEBUG'})
     assert response == {'status': 200, 'name': 'sqlalchemy.engine', 'level': 'DEBUG',
                         'effective_level': 'DEBUG'}
 
 
 def test_api_bad_secret(app_connection):
-    app_connection.get_json('logging/level', params={'secret': 'wrong', 'name': 'sqlalchemy.engine'},
+    app_connection.get_json('c2c/logging/level', params={'secret': 'wrong', 'name': 'sqlalchemy.engine'},
                             expected_status=403)
 
 
 def test_api_missing_secret(app_connection):
-    app_connection.get_json('logging/level', params={'name': 'sqlalchemy.engine'}, expected_status=403)
+    app_connection.get_json('c2c/logging/level', params={'name': 'sqlalchemy.engine'}, expected_status=403)

acceptance_tests/tests/tests/test_sql_profiler.py

@@ -2,7 +2,7 @@ def _switch(app_connection, enable=None):
     params = {'secret': 'changeme'}
     if enable is not None:
         params['enable'] = "1" if enable else "0"
-    answer = app_connection.get_json("sql_profiler", params=params)
+    answer = app_connection.get_json("c2c/sql_profiler", params=params)
     assert answer['status'] == 200
     return answer['enabled']
 
@@ -18,4 +18,4 @@ def test_ok(app_connection, slave_db_connection):
 
 
 def test_no_secret(app_connection):
-    app_connection.get_json("sql_profiler", params={'enable': '1'}, expected_status=403)
+    app_connection.get_json("c2c/sql_profiler", params={'enable': '1'}, expected_status=403)

acceptance_tests/tests/tests/test_stats.py

@@ -1,10 +1,10 @@
 def test_ok(app_connection):
     # reset the stats to be sure where we are at
-    app_connection.get_json('stats.json?reset=1', cors=False)
+    app_connection.get_json('c2c/stats.json?reset=1', cors=False)
 
     app_connection.get_json("hello")  # to be sure we have some stats
 
-    stats = app_connection.get_json('stats.json', cors=False)
+    stats = app_connection.get_json('c2c/stats.json', cors=False)
     print(stats)
     assert stats['timers']['render/GET/hello/200']['nb'] == 1
     assert stats['timers']['route/GET/hello/200']['nb'] == 1

acceptance_tests/tests/tests/test_versions.py

@@ -1,5 +1,5 @@
 def test_ok(app_connection):
-    response = app_connection.get_json('versions.json')
+    response = app_connection.get_json('c2c/versions.json')
     assert 'main' in response
     assert 'git_hash' in response['main']
     assert 'packages' in response

c2cwsgiutils/_utils.py

@@ -0,0 +1,15 @@
+"""
+Private utilities.
+"""
+
+import os
+
+
+def get_base_path(config):
+    return env_or_config(config, 'C2C_BASE_PATH', 'c2c.base_path', '')
+
+
+def env_or_config(config, env_name, config_name, default):
+    if env_name in os.environ:
+        return os.environ[env_name]
+    return config.get_settings().get(config_name, default)

c2cwsgiutils/db.py

@@ -16,7 +16,7 @@ class Tweens(object):
 tweens = Tweens()
 
 
-def setup_session(config, master_prefix, slave_prefix="", force_master=None, force_slave=None):
+def setup_session(config, master_prefix, slave_prefix=None, force_master=None, force_slave=None):
     """
     Create a SQLAlchemy session with an accompanying tween that switches between the master and the slave
     DB connection.
@@ -51,10 +51,10 @@ def db_chooser_tween(request):
             has_force_master = any(r.match(method_path) for r in master_paths)
             if not has_force_master and (request.method in ("GET", "OPTIONS") or
                                          any(r.match(method_path) for r in slave_paths)):
-                LOG.debug("Using slave database for: %s", method_path)
+                LOG.debug("Using %s database for: %s", slave_prefix, method_path)
                 session.bind = ro_engine
             else:
-                LOG.debug("Using master database for: %s", method_path)
+                LOG.debug("Using %s database for: %s", master_prefix, method_path)
                 session.bind = rw_engine
 
             try:
@@ -64,6 +64,8 @@ def db_chooser_tween(request):
 
         return db_chooser_tween
 
+    if slave_prefix is None:
+        slave_prefix = master_prefix
     settings = config.registry.settings
     rw_engine = sqlalchemy.engine_from_config(settings, master_prefix + ".")
 
@@ -79,4 +81,9 @@ def db_chooser_tween(request):
 
     db_session = sqlalchemy.orm.scoped_session(
         sqlalchemy.orm.sessionmaker(extension=ZopeTransactionExtension(), bind=rw_engine))
+    if settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"):
+        db_session.c2c_rw_bind = rw_engine
+        db_session.c2c_ro_bind = ro_engine
+    rw_engine.c2c_name = master_prefix
+    ro_engine.c2c_name = slave_prefix
     return db_session, rw_engine, ro_engine

c2cwsgiutils/health_check.py

@@ -0,0 +1,105 @@
+"""
+Setup an health_check API.
+
+To use it, create an instance of this class in your application initialization and do a few calls to its
+methods add_db_check()
+"""
+import logging
+import os
+import traceback
+
+import requests
+from c2cwsgiutils import stats, _utils
+from pyramid.httpexceptions import HTTPNotFound
+
+LOG = logging.getLogger(__name__)
+
+
+class HealthCheck:
+    def __init__(self, config):
+        config.add_route("c2c_health_check", _utils.get_base_path(config) + r"/health_check",
+                         request_method="GET")
+        config.add_view(self._view, route_name="c2c_health_check", renderer="json", http_cache=0)
+        self._checks = []
+
+    def add_db_session_check(self, session, query_cb=None, at_least_one_model=None, level=1):
+        """
+        Check a DB session is working. You can specify either query_cb or at_least_one_model.
+        :param session: a DB session created by c2cwsgiutils.db.setup_session()
+        :param query_cb: a callable that take a session as parameter and check it works
+        :param at_least_one_model: a model that must have at least one entry in the DB
+        :param level: the level of the health check
+        """
+        if query_cb is None:
+            query_cb = self._at_least_one(at_least_one_model)
+        self._checks.append(self._create_db_engine_check(session, session.c2c_rw_bind, query_cb) + (level,))
+        if session.c2c_rw_bind != session.c2c_ro_bind:
+            self._checks.append(self._create_db_engine_check(session, session.c2c_ro_bind,
+                                                             query_cb) + (level,))
+
+    def add_url_check(self, url, name=None, check_cb=lambda request, response: None, timeout=3, level=1):
+        """
+        Check that a GET on an URL returns 2xx
+        :param url: the URL to query
+        :param name: the name of the check (defaults to url)
+        :param check_cb: an optional CB to do additional checks on the response (takes the request and the
+                         response as parameters)
+        :param timeout: the timeout
+        :param level: the level of the health check
+        """
+        def check(request):
+            response = requests.get(url, timeout=timeout)
+            response.raise_for_status()
+            check_cb(request, response)
+        if name is None:
+            name = url
+        self._checks.append((name, check, level))
+
+    def add_custom_check(self, name, check_cb, level):
+        """
+        Add a custom check
+        :param name: the name of the check
+        :param check_cb: the callback to call (takes the request as parameter)
+        :param level: the level of the health check
+        """
+        self._checks.append((name, check_cb, level))
+
+    def _view(self, request):
+        max_level = int(request.params.get('max_level', '1'))
+        successes = []
+        failures = {}
+        for name, check, level in self._checks:
+            if level <= max_level:
+                try:
+                    check(request)
+                    successes.append(name)
+                except Exception as e:
+                    trace = traceback.format_exc()
+                    LOG.error(trace)
+                    failure = {'message': str(e)}
+                    if os.environ.get('DEVELOPMENT', '0') != '0':
+                        failure['stacktrace'] = trace
+                    failures[name] = failure
+
+        if failures:
+            request.response.status = 500
+
+        return {'status': 500 if failures else 200, 'failures': failures, 'successes': successes}
+
+    def _create_db_engine_check(self, session, bind, query_cb):
+        def check(request):
+            prev_bind = session.bind
+            try:
+                session.bind = bind
+                with stats.timer_context(['sql', 'manual', 'health_check',  'db', bind.c2c_name]):
+                    query_cb(session)
+            finally:
+                session.bind = prev_bind
+        return 'db_engine_' + bind.c2c_name, check
+
+    def _at_least_one(self, model):
+        def query(session):
+            result = session.query(model).first()
+            if result is None:
+                raise HTTPNotFound(model.__name__ + " record not found")
+        return query

c2cwsgiutils/pyramid_logging.py

@@ -15,6 +15,8 @@
 import pyramid.events
 from pyramid.httpexceptions import HTTPForbidden
 
+from c2cwsgiutils import _utils
+
 LOG = logging.getLogger(__name__)
 
 
@@ -63,8 +65,9 @@ def install_subscriber(config):
     config.add_subscriber(_set_context, pyramid.events.NewRequest)
 
     if 'LOG_VIEW_SECRET' in os.environ:
-        config.add_route("logging_level", r"/logging/level", request_method="GET")
-        config.add_view(_logging_change_level, route_name="logging_level", renderer="json", http_cache=0)
+        config.add_route("c2c_logging_level", _utils.get_base_path(config) + r"/logging/level",
+                         request_method="GET")
+        config.add_view(_logging_change_level, route_name="c2c_logging_level", renderer="json", http_cache=0)
         LOG.info("Enabled the /logging/change_level API")
 
 

c2cwsgiutils/sql_profiler.py

@@ -9,6 +9,8 @@
 import sqlalchemy.event
 import sqlalchemy.engine
 
+from c2cwsgiutils import _utils
+
 ENV_KEY = 'SQL_PROFILER_SECRET'
 LOG = logging.getLogger(__name__)
 enabled = False
@@ -67,6 +69,7 @@ def init(config):
     Install a pyramid  event handler that adds the request information
     """
     if 'SQL_PROFILER_SECRET' in os.environ:
-        config.add_route("sql_profiler", r"/sql_profiler", request_method="GET")
-        config.add_view(_sql_profiler_view, route_name="sql_profiler", renderer="json", http_cache=0)
+        config.add_route("c2c_sql_profiler", _utils.get_base_path(config) + r"/sql_profiler",
+                         request_method="GET")
+        config.add_view(_sql_profiler_view, route_name="c2c_sql_profiler", renderer="json", http_cache=0)
         LOG.info("Enabled the /sql_profiler API")

c2cwsgiutils/stats.py

@@ -14,6 +14,8 @@
 from pyramid.httpexceptions import HTTPException
 import sqlalchemy.event
 
+from c2cwsgiutils import _utils
+
 BACKENDS = []
 LOG = logging.getLogger(__name__)
 
@@ -276,8 +278,10 @@ def init_backends(config):
         memory_backend = _MemoryBackend()
         BACKENDS.append(memory_backend)
 
-        config.add_route("read_stats_json", r"/stats.json", request_method="GET")
-        config.add_view(memory_backend.get_stats, route_name="read_stats_json", renderer="json", http_cache=0)
+        config.add_route("c2c_read_stats_json", _utils.get_base_path(config) + r"/stats.json",
+                         request_method="GET")
+        config.add_view(memory_backend.get_stats, route_name="c2c_read_stats_json", renderer="json",
+                        http_cache=0)
 
     statsd_address = _get_env_or_settings(config, "STATSD_ADDRESS", "statsd_address", None)
     if statsd_address is not None:  # pragma: nocover