From 91cb6c986540b3f3c1521d9595778bf91c6ccb93 Mon Sep 17 00:00:00 2001 From: Patrick Valsecchi Date: Thu, 6 Apr 2017 16:31:17 +0200 Subject: [PATCH] Add health_check framework Add the possibility to add a prefix to all the URLs managed by c2cwsgiutils. Just define `c2c.base_path` in the production.ini or the `C2C_BASE_PATH` environment variable. --- .gitignore | 1 + README.md | 1 + .../app/c2cwsgiutils_app/__init__.py | 11 ++ acceptance_tests/app/production.ini | 2 + .../tests/tests/test_health_check.py | 36 ++++++ acceptance_tests/tests/tests/test_logging.py | 16 +-- .../tests/tests/test_sql_profiler.py | 4 +- acceptance_tests/tests/tests/test_stats.py | 4 +- acceptance_tests/tests/tests/test_versions.py | 2 +- c2cwsgiutils/_utils.py | 15 +++ c2cwsgiutils/db.py | 13 ++- c2cwsgiutils/health_check.py | 105 ++++++++++++++++++ c2cwsgiutils/pyramid_logging.py | 7 +- c2cwsgiutils/sql_profiler.py | 7 +- c2cwsgiutils/stats.py | 8 +- c2cwsgiutils/version.py | 5 +- setup.py | 2 +- 17 files changed, 215 insertions(+), 24 deletions(-) create mode 100644 acceptance_tests/tests/tests/test_health_check.py create mode 100644 c2cwsgiutils/_utils.py create mode 100644 c2cwsgiutils/health_check.py diff --git a/.gitignore b/.gitignore index 7422f5dda..05fda24f6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ /.venv/ /acceptance_tests/app/c2cwsgiutils /acceptance_tests/app/c2cwsgiutils_run +/acceptance_tests/app/c2cwsgiutils_genversion.py /acceptance_tests/app/rel_requirements.txt /acceptance_tests/app/setup.cfg /acceptance_tests/tests/c2cwsgiutils diff --git a/README.md b/README.md index 28ede712c..f1e9c511f 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ applications: SQL_PROFILER_SECRET env var and using the view (/sql_profiler) to switch it ON and OFF. Warning, it will slow down everything. * A view to get the version information about the application and the installed packages (/versions.json) +* A framework for implementing a health_check service (/health_check) * Error handlers to send JSON messages to the client in case of error * A cornice service drop in replacement for setting up CORS diff --git a/acceptance_tests/app/c2cwsgiutils_app/__init__.py b/acceptance_tests/app/c2cwsgiutils_app/__init__.py index 087a22f7e..bb6812e64 100644 --- a/acceptance_tests/app/c2cwsgiutils_app/__init__.py +++ b/acceptance_tests/app/c2cwsgiutils_app/__init__.py @@ -1,9 +1,16 @@ + import c2cwsgiutils.pyramid +from c2cwsgiutils.health_check import HealthCheck from pyramid.config import Configurator +from pyramid.httpexceptions import HTTPInternalServerError from c2cwsgiutils_app import models +def _failure(_request): + raise HTTPInternalServerError('failing check') + + def main(_, **settings): """ This function returns a Pyramid WSGI application. """ @@ -11,5 +18,9 @@ def main(_, **settings): config.include(c2cwsgiutils.pyramid.includeme) models.init(config) config.scan("c2cwsgiutils_app.services") + health_check = HealthCheck(config) + health_check.add_db_session_check(models.DBSession, at_least_one_model=models.Hello) + health_check.add_url_check('http://localhost/api/hello') + health_check.add_custom_check('fail', _failure, 2) return config.make_wsgi_app() diff --git a/acceptance_tests/app/production.ini b/acceptance_tests/app/production.ini index 222f32e48..36e1413ae 100644 --- a/acceptance_tests/app/production.ini +++ b/acceptance_tests/app/production.ini @@ -22,6 +22,8 @@ sqlalchemy_slave.pool_recycle = 30 sqlalchemy_slave.pool_size = 5 sqlalchemy_slave.max_overflow = 25 +c2c.base_path = /c2c + ### # logging configuration # http://docs.pylonsproject.org/projects/pyramid/en/1.6-branch/narr/logging.html diff --git a/acceptance_tests/tests/tests/test_health_check.py b/acceptance_tests/tests/tests/test_health_check.py new file mode 100644 index 000000000..c73bdf9d9 --- /dev/null +++ b/acceptance_tests/tests/tests/test_health_check.py @@ -0,0 +1,36 @@ +import json + + +def test_ok(app_connection): + response = app_connection.get_json("c2c/health_check") + print('response=' + json.dumps(response)) + assert response == { + 'status': 200, + 'successes': ['db_engine_sqlalchemy', 'db_engine_sqlalchemy_slave', 'http://localhost/api/hello'], + 'failures': {} + } + + +def test_failure(app_connection): + response = app_connection.get_json("c2c/health_check", params={'max_level': '2'}, expected_status=500) + print('response=' + json.dumps(response)) + assert response == { + 'status': 500, + 'successes': ['db_engine_sqlalchemy', 'db_engine_sqlalchemy_slave', 'http://localhost/api/hello'], + 'failures': { + 'fail': { + 'message': 'failing check', + 'stacktrace': response['failures']['fail']['stacktrace'] + } + } + } + + +def test_ping(app_connection): + response = app_connection.get_json("c2c/health_check", params={'max_level': '0'}) + print('response=' + json.dumps(response)) + assert response == { + 'status': 200, + 'successes': [], + 'failures': {} + } diff --git a/acceptance_tests/tests/tests/test_logging.py b/acceptance_tests/tests/tests/test_logging.py index 13bbf00e8..9922dd19f 100644 --- a/acceptance_tests/tests/tests/test_logging.py +++ b/acceptance_tests/tests/tests/test_logging.py @@ -1,31 +1,31 @@ def test_api(app_connection): - response = app_connection.get_json('logging/level', + response = app_connection.get_json('c2c/logging/level', params={'secret': 'changeme', 'name': 'sqlalchemy.engine'}) assert response == {'status': 200, 'name': 'sqlalchemy.engine', 'level': 'DEBUG', 'effective_level': 'DEBUG'} - response = app_connection.get_json('logging/level', + response = app_connection.get_json('c2c/logging/level', params={'secret': 'changeme', 'name': 'sqlalchemy.engine.sub'}) assert response == {'status': 200, 'name': 'sqlalchemy.engine.sub', 'level': 'NOTSET', 'effective_level': 'DEBUG'} - response = app_connection.get_json('logging/level', + response = app_connection.get_json('c2c/logging/level', params={'secret': 'changeme', 'name': 'sqlalchemy.engine', 'level': 'INFO'}) assert response == {'status': 200, 'name': 'sqlalchemy.engine', 'level': 'INFO', 'effective_level': 'INFO'} - response = app_connection.get_json('logging/level', + response = app_connection.get_json('c2c/logging/level', params={'secret': 'changeme', 'name': 'sqlalchemy.engine'}) assert response == {'status': 200, 'name': 'sqlalchemy.engine', 'level': 'INFO', 'effective_level': 'INFO'} - response = app_connection.get_json('logging/level', + response = app_connection.get_json('c2c/logging/level', params={'secret': 'changeme', 'name': 'sqlalchemy.engine.sub'}) assert response == {'status': 200, 'name': 'sqlalchemy.engine.sub', 'level': 'NOTSET', 'effective_level': 'INFO'} - response = app_connection.get_json('logging/level', + response = app_connection.get_json('c2c/logging/level', params={'secret': 'changeme', 'name': 'sqlalchemy.engine', 'level': 'DEBUG'}) assert response == {'status': 200, 'name': 'sqlalchemy.engine', 'level': 'DEBUG', @@ -33,9 +33,9 @@ def test_api(app_connection): def test_api_bad_secret(app_connection): - app_connection.get_json('logging/level', params={'secret': 'wrong', 'name': 'sqlalchemy.engine'}, + app_connection.get_json('c2c/logging/level', params={'secret': 'wrong', 'name': 'sqlalchemy.engine'}, expected_status=403) def test_api_missing_secret(app_connection): - app_connection.get_json('logging/level', params={'name': 'sqlalchemy.engine'}, expected_status=403) + app_connection.get_json('c2c/logging/level', params={'name': 'sqlalchemy.engine'}, expected_status=403) diff --git a/acceptance_tests/tests/tests/test_sql_profiler.py b/acceptance_tests/tests/tests/test_sql_profiler.py index 19107fb83..af3fcc50b 100644 --- a/acceptance_tests/tests/tests/test_sql_profiler.py +++ b/acceptance_tests/tests/tests/test_sql_profiler.py @@ -2,7 +2,7 @@ def _switch(app_connection, enable=None): params = {'secret': 'changeme'} if enable is not None: params['enable'] = "1" if enable else "0" - answer = app_connection.get_json("sql_profiler", params=params) + answer = app_connection.get_json("c2c/sql_profiler", params=params) assert answer['status'] == 200 return answer['enabled'] @@ -18,4 +18,4 @@ def test_ok(app_connection, slave_db_connection): def test_no_secret(app_connection): - app_connection.get_json("sql_profiler", params={'enable': '1'}, expected_status=403) + app_connection.get_json("c2c/sql_profiler", params={'enable': '1'}, expected_status=403) diff --git a/acceptance_tests/tests/tests/test_stats.py b/acceptance_tests/tests/tests/test_stats.py index bf106839f..19075042b 100644 --- a/acceptance_tests/tests/tests/test_stats.py +++ b/acceptance_tests/tests/tests/test_stats.py @@ -1,10 +1,10 @@ def test_ok(app_connection): # reset the stats to be sure where we are at - app_connection.get_json('stats.json?reset=1', cors=False) + app_connection.get_json('c2c/stats.json?reset=1', cors=False) app_connection.get_json("hello") # to be sure we have some stats - stats = app_connection.get_json('stats.json', cors=False) + stats = app_connection.get_json('c2c/stats.json', cors=False) print(stats) assert stats['timers']['render/GET/hello/200']['nb'] == 1 assert stats['timers']['route/GET/hello/200']['nb'] == 1 diff --git a/acceptance_tests/tests/tests/test_versions.py b/acceptance_tests/tests/tests/test_versions.py index 34b9c389a..08561f484 100644 --- a/acceptance_tests/tests/tests/test_versions.py +++ b/acceptance_tests/tests/tests/test_versions.py @@ -1,5 +1,5 @@ def test_ok(app_connection): - response = app_connection.get_json('versions.json') + response = app_connection.get_json('c2c/versions.json') assert 'main' in response assert 'git_hash' in response['main'] assert 'packages' in response diff --git a/c2cwsgiutils/_utils.py b/c2cwsgiutils/_utils.py new file mode 100644 index 000000000..3f6b11f18 --- /dev/null +++ b/c2cwsgiutils/_utils.py @@ -0,0 +1,15 @@ +""" +Private utilities. +""" + +import os + + +def get_base_path(config): + return env_or_config(config, 'C2C_BASE_PATH', 'c2c.base_path', '') + + +def env_or_config(config, env_name, config_name, default): + if env_name in os.environ: + return os.environ[env_name] + return config.get_settings().get(config_name, default) diff --git a/c2cwsgiutils/db.py b/c2cwsgiutils/db.py index 9c416d14b..31f3a51ac 100644 --- a/c2cwsgiutils/db.py +++ b/c2cwsgiutils/db.py @@ -16,7 +16,7 @@ class Tweens(object): tweens = Tweens() -def setup_session(config, master_prefix, slave_prefix="", force_master=None, force_slave=None): +def setup_session(config, master_prefix, slave_prefix=None, force_master=None, force_slave=None): """ Create a SQLAlchemy session with an accompanying tween that switches between the master and the slave DB connection. @@ -51,10 +51,10 @@ def db_chooser_tween(request): has_force_master = any(r.match(method_path) for r in master_paths) if not has_force_master and (request.method in ("GET", "OPTIONS") or any(r.match(method_path) for r in slave_paths)): - LOG.debug("Using slave database for: %s", method_path) + LOG.debug("Using %s database for: %s", slave_prefix, method_path) session.bind = ro_engine else: - LOG.debug("Using master database for: %s", method_path) + LOG.debug("Using %s database for: %s", master_prefix, method_path) session.bind = rw_engine try: @@ -64,6 +64,8 @@ def db_chooser_tween(request): return db_chooser_tween + if slave_prefix is None: + slave_prefix = master_prefix settings = config.registry.settings rw_engine = sqlalchemy.engine_from_config(settings, master_prefix + ".") @@ -79,4 +81,9 @@ def db_chooser_tween(request): db_session = sqlalchemy.orm.scoped_session( sqlalchemy.orm.sessionmaker(extension=ZopeTransactionExtension(), bind=rw_engine)) + if settings[master_prefix + ".url"] != settings.get(slave_prefix + ".url"): + db_session.c2c_rw_bind = rw_engine + db_session.c2c_ro_bind = ro_engine + rw_engine.c2c_name = master_prefix + ro_engine.c2c_name = slave_prefix return db_session, rw_engine, ro_engine diff --git a/c2cwsgiutils/health_check.py b/c2cwsgiutils/health_check.py new file mode 100644 index 000000000..9babc833b --- /dev/null +++ b/c2cwsgiutils/health_check.py @@ -0,0 +1,105 @@ +""" +Setup an health_check API. + +To use it, create an instance of this class in your application initialization and do a few calls to its +methods add_db_check() +""" +import logging +import os +import traceback + +import requests +from c2cwsgiutils import stats, _utils +from pyramid.httpexceptions import HTTPNotFound + +LOG = logging.getLogger(__name__) + + +class HealthCheck: + def __init__(self, config): + config.add_route("c2c_health_check", _utils.get_base_path(config) + r"/health_check", + request_method="GET") + config.add_view(self._view, route_name="c2c_health_check", renderer="json", http_cache=0) + self._checks = [] + + def add_db_session_check(self, session, query_cb=None, at_least_one_model=None, level=1): + """ + Check a DB session is working. You can specify either query_cb or at_least_one_model. + :param session: a DB session created by c2cwsgiutils.db.setup_session() + :param query_cb: a callable that take a session as parameter and check it works + :param at_least_one_model: a model that must have at least one entry in the DB + :param level: the level of the health check + """ + if query_cb is None: + query_cb = self._at_least_one(at_least_one_model) + self._checks.append(self._create_db_engine_check(session, session.c2c_rw_bind, query_cb) + (level,)) + if session.c2c_rw_bind != session.c2c_ro_bind: + self._checks.append(self._create_db_engine_check(session, session.c2c_ro_bind, + query_cb) + (level,)) + + def add_url_check(self, url, name=None, check_cb=lambda request, response: None, timeout=3, level=1): + """ + Check that a GET on an URL returns 2xx + :param url: the URL to query + :param name: the name of the check (defaults to url) + :param check_cb: an optional CB to do additional checks on the response (takes the request and the + response as parameters) + :param timeout: the timeout + :param level: the level of the health check + """ + def check(request): + response = requests.get(url, timeout=timeout) + response.raise_for_status() + check_cb(request, response) + if name is None: + name = url + self._checks.append((name, check, level)) + + def add_custom_check(self, name, check_cb, level): + """ + Add a custom check + :param name: the name of the check + :param check_cb: the callback to call (takes the request as parameter) + :param level: the level of the health check + """ + self._checks.append((name, check_cb, level)) + + def _view(self, request): + max_level = int(request.params.get('max_level', '1')) + successes = [] + failures = {} + for name, check, level in self._checks: + if level <= max_level: + try: + check(request) + successes.append(name) + except Exception as e: + trace = traceback.format_exc() + LOG.error(trace) + failure = {'message': str(e)} + if os.environ.get('DEVELOPMENT', '0') != '0': + failure['stacktrace'] = trace + failures[name] = failure + + if failures: + request.response.status = 500 + + return {'status': 500 if failures else 200, 'failures': failures, 'successes': successes} + + def _create_db_engine_check(self, session, bind, query_cb): + def check(request): + prev_bind = session.bind + try: + session.bind = bind + with stats.timer_context(['sql', 'manual', 'health_check', 'db', bind.c2c_name]): + query_cb(session) + finally: + session.bind = prev_bind + return 'db_engine_' + bind.c2c_name, check + + def _at_least_one(self, model): + def query(session): + result = session.query(model).first() + if result is None: + raise HTTPNotFound(model.__name__ + " record not found") + return query diff --git a/c2cwsgiutils/pyramid_logging.py b/c2cwsgiutils/pyramid_logging.py index cebccc0d6..9c87993f7 100644 --- a/c2cwsgiutils/pyramid_logging.py +++ b/c2cwsgiutils/pyramid_logging.py @@ -15,6 +15,8 @@ import pyramid.events from pyramid.httpexceptions import HTTPForbidden +from c2cwsgiutils import _utils + LOG = logging.getLogger(__name__) @@ -63,8 +65,9 @@ def install_subscriber(config): config.add_subscriber(_set_context, pyramid.events.NewRequest) if 'LOG_VIEW_SECRET' in os.environ: - config.add_route("logging_level", r"/logging/level", request_method="GET") - config.add_view(_logging_change_level, route_name="logging_level", renderer="json", http_cache=0) + config.add_route("c2c_logging_level", _utils.get_base_path(config) + r"/logging/level", + request_method="GET") + config.add_view(_logging_change_level, route_name="c2c_logging_level", renderer="json", http_cache=0) LOG.info("Enabled the /logging/change_level API") diff --git a/c2cwsgiutils/sql_profiler.py b/c2cwsgiutils/sql_profiler.py index 0eb3098c7..f099536e4 100644 --- a/c2cwsgiutils/sql_profiler.py +++ b/c2cwsgiutils/sql_profiler.py @@ -9,6 +9,8 @@ import sqlalchemy.event import sqlalchemy.engine +from c2cwsgiutils import _utils + ENV_KEY = 'SQL_PROFILER_SECRET' LOG = logging.getLogger(__name__) enabled = False @@ -67,6 +69,7 @@ def init(config): Install a pyramid event handler that adds the request information """ if 'SQL_PROFILER_SECRET' in os.environ: - config.add_route("sql_profiler", r"/sql_profiler", request_method="GET") - config.add_view(_sql_profiler_view, route_name="sql_profiler", renderer="json", http_cache=0) + config.add_route("c2c_sql_profiler", _utils.get_base_path(config) + r"/sql_profiler", + request_method="GET") + config.add_view(_sql_profiler_view, route_name="c2c_sql_profiler", renderer="json", http_cache=0) LOG.info("Enabled the /sql_profiler API") diff --git a/c2cwsgiutils/stats.py b/c2cwsgiutils/stats.py index 8549bebac..6f39791eb 100644 --- a/c2cwsgiutils/stats.py +++ b/c2cwsgiutils/stats.py @@ -14,6 +14,8 @@ from pyramid.httpexceptions import HTTPException import sqlalchemy.event +from c2cwsgiutils import _utils + BACKENDS = [] LOG = logging.getLogger(__name__) @@ -276,8 +278,10 @@ def init_backends(config): memory_backend = _MemoryBackend() BACKENDS.append(memory_backend) - config.add_route("read_stats_json", r"/stats.json", request_method="GET") - config.add_view(memory_backend.get_stats, route_name="read_stats_json", renderer="json", http_cache=0) + config.add_route("c2c_read_stats_json", _utils.get_base_path(config) + r"/stats.json", + request_method="GET") + config.add_view(memory_backend.get_stats, route_name="c2c_read_stats_json", renderer="json", + http_cache=0) statsd_address = _get_env_or_settings(config, "STATSD_ADDRESS", "statsd_address", None) if statsd_address is not None: # pragma: nocover diff --git a/c2cwsgiutils/version.py b/c2cwsgiutils/version.py index d76897519..10b212879 100644 --- a/c2cwsgiutils/version.py +++ b/c2cwsgiutils/version.py @@ -2,6 +2,8 @@ import json import os +from c2cwsgiutils import _utils + VERSIONS_PATH = '/app/versions.json' LOG = logging.getLogger(__name__) @@ -10,6 +12,7 @@ def init(config): if os.path.isfile(VERSIONS_PATH): with open(VERSIONS_PATH) as file: versions = json.load(file) - config.add_route("c2c_versions", r"/versions.json", request_method="GET") + config.add_route("c2c_versions", _utils.get_base_path(config) + r"/versions.json", + request_method="GET") config.add_view(lambda request: versions, route_name="c2c_versions", renderer="json", http_cache=0) LOG.info("Installed the /versions.json service") diff --git a/setup.py b/setup.py index 37e966c26..639d7a1fe 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import setup, find_packages -VERSION = '0.8.0' +VERSION = '0.9.0' HERE = os.path.abspath(os.path.dirname(__file__)) INSTALL_REQUIRES = open(os.path.join(HERE, 'rel_requirements.txt')).read().splitlines()