From 6976e5d6cf9353b8753e07813166d4e9b5a0919c Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Mon, 3 Oct 2022 08:20:33 +0100 Subject: [PATCH 1/2] Update logging cookbook with an example on custom handling of levels. --- Doc/howto/logging-cookbook.rst | 210 ++++++++++++++++++++++++++++++++- 1 file changed, 206 insertions(+), 4 deletions(-) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 99e886c61b4c5f..2187e2ef645a34 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -276,6 +276,211 @@ choose a different directory name for the log - just ensure that the directory e and that you have the permissions to create and update files in it. +.. _custom-level-handling: + +Custom handling of levels +------------------------- + +Sometimes, you might want to do something slightly different from the standard +handling of levels in handlers, where all levels above a threshold get +processed by a handler. To do this, you need to use filters. Let's look at s +scenario where you want to arrange things as follows: + +* Send messages of severity ``INFO`` and ``WARNING`` to ``sys.stdout`` +* Send messages of severity ``ERROR`` and above to ``sys.stderr`` +* Send messages of severity ``DEBUG`` and above to file ``app.log`` + +Suppose you configure logging with the following JSON: + +.. code-block:: json + + { + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(levelname)-8s - %(message)s" + } + }, + "handlers": { + "stdout": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "simple", + "stream": "ext://sys.stdout", + }, + "stderr": { + "class": "logging.StreamHandler", + "level": "ERROR", + "formatter": "simple", + "stream": "ext://sys.stderr" + }, + "file": { + "class": "logging.FileHandler", + "formatter": "simple", + "filename": "app.log", + "mode": "w" + } + }, + "root": { + "level": "DEBUG", + "handlers": [ + "stderr", + "stdout", + "file" + ] + } + } + +This configuration does *almost* what we want, except that ``sys.stdout`` would +show messages of severity ``ERROR`` and above as well as ``INFO`` and +``WARNING`` messages. To prevent this, we can set up a filter which excludes +those messages and add it to the relevant handler. This can be configured by +adding a ``filters`` section parallel to ``formatters`` and ``handlers``: + +.. code-block:: json + + "filters": { + "warnings_and_below": { + "()" : "__main__.filter_maker", + "level": "WARNING" + } + } + +and changing the section on the ``stdout`` handler to add it: + +.. code-block:: json + + "stdout": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "simple", + "stream": "ext://sys.stdout", + "filters": ["warnings_and_below"] + } + +A filter is just a function, so we can define the ``filter_maker`` (a factory +function) as follows: + +.. code-block:: python + + def filter_maker(level): + level = getattr(logging, level) + + def filter(record): + return record.levelno <= level + + return filter + +This converts the string argument passed in to a numeric level, and returns a +function which only returns ``True`` if the level of the passed in record is +at or below the specified level. Note that in this example I have defined the +``filter_maker`` in a test script ``main.py`` that I run from the command line, +so its module will be ``__main__`` - hence the ``__main__.filter_maker`` in the +filter configuration. You will need to change that if you define it in a +different module. + +With the filter added, we can run ``main.py``, which in full is: + +.. code-block:: python + + import json + import logging + import logging.config + + CONFIG = ''' + { + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "simple": { + "format": "%(levelname)-8s - %(message)s" + } + }, + "filters": { + "warnings_and_below": { + "()" : "__main__.filter_maker", + "level": "WARNING" + } + }, + "handlers": { + "stdout": { + "class": "logging.StreamHandler", + "level": "INFO", + "formatter": "simple", + "stream": "ext://sys.stdout", + "filters": ["warnings_and_below"] + }, + "stderr": { + "class": "logging.StreamHandler", + "level": "ERROR", + "formatter": "simple", + "stream": "ext://sys.stderr" + }, + "file": { + "class": "logging.FileHandler", + "formatter": "simple", + "filename": "app.log", + "mode": "w" + } + }, + "root": { + "level": "DEBUG", + "handlers": [ + "stderr", + "stdout", + "file" + ] + } + } + ''' + + def filter_maker(level): + level = getattr(logging, level) + + def filter(record): + return record.levelno <= level + + return filter + + logging.config.dictConfig(json.loads(CONFIG)) + logging.debug('A DEBUG message') + logging.info('An INFO message') + logging.warning('A WARNING message') + logging.error('An ERROR message') + logging.critical('A CRITICAL message') + +And after running it like this: + +.. code-block:: shell + + python main.py 2>stderr.log >stdout.log + +We can see the results are as expected: + +.. code-block:: shell + + $ more *.log + :::::::::::::: + app.log + :::::::::::::: + DEBUG - A DEBUG message + INFO - An INFO message + WARNING - A WARNING message + ERROR - An ERROR message + CRITICAL - A CRITICAL message + :::::::::::::: + stderr.log + :::::::::::::: + ERROR - An ERROR message + CRITICAL - A CRITICAL message + :::::::::::::: + stdout.log + :::::::::::::: + INFO - An INFO message + WARNING - A WARNING message + + Configuration server example ---------------------------- @@ -3503,7 +3708,7 @@ instance). Then, you'd get this kind of result: WARNING:demo:Bar >>> -Of course, these above examples show output according to the format used by +Of course, the examples above show output according to the format used by :func:`~logging.basicConfig`, but you can use a different formatter when you configure logging. @@ -3517,7 +3722,6 @@ need to do or deal with, it is worth mentioning some usage patterns which are *unhelpful*, and which should therefore be avoided in most cases. The following sections are in no particular order. - Opening the same log file multiple times ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -3566,7 +3770,6 @@ that in other languages such as Java and C#, loggers are often static class attributes. However, this pattern doesn't make sense in Python, where the module (and not the class) is the unit of software decomposition. - Adding handlers other than :class:`NullHandler` to a logger in a library ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -3575,7 +3778,6 @@ responsibility of the application developer, not the library developer. If you are maintaining a library, ensure that you don't add handlers to any of your loggers other than a :class:`~logging.NullHandler` instance. - Creating a lot of loggers ^^^^^^^^^^^^^^^^^^^^^^^^^ From 6cd7ca2b1544c4e823e6236d64c833db389943f1 Mon Sep 17 00:00:00 2001 From: Vinay Sajip Date: Sat, 15 Oct 2022 20:15:53 +0100 Subject: [PATCH 2/2] Fix typo - thanks, Jelle. Co-authored-by: Jelle Zijlstra --- Doc/howto/logging-cookbook.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Doc/howto/logging-cookbook.rst b/Doc/howto/logging-cookbook.rst index 2187e2ef645a34..eac34aaab3aa53 100644 --- a/Doc/howto/logging-cookbook.rst +++ b/Doc/howto/logging-cookbook.rst @@ -283,7 +283,7 @@ Custom handling of levels Sometimes, you might want to do something slightly different from the standard handling of levels in handlers, where all levels above a threshold get -processed by a handler. To do this, you need to use filters. Let's look at s +processed by a handler. To do this, you need to use filters. Let's look at a scenario where you want to arrange things as follows: * Send messages of severity ``INFO`` and ``WARNING`` to ``sys.stdout``