From b773f71e3dc2548636bb43f59c8b1c0f882de3d9 Mon Sep 17 00:00:00 2001 From: PaulNewtech <62654954+PaulNewtech@users.noreply.github.com> Date: Fri, 5 Nov 2021 08:56:19 +0100 Subject: [PATCH] Handle syslog writing from the command line (#9513) * Handle syslog writing from the command line * add an arg in the command line to use syslog * respect standards * line was too long * new arg for command line * add sanic logs to syslog * del useless statement * quick fix * too many blank line * Use default communication with syslog * possibility to use a syslog server * fix indent * Fix for constants access * fix typo Co-authored-by: Joe Juzl * fix typo * fix for constant definition in bad file * Black linter compliant * Function add_server_arguments had 26 lines of code (exceeds 25 allowed). * fix for bad merge * for lint compliance * add required param * adapt test for log enhancement Co-authored-by: Joe Juzl --- rasa/cli/arguments/run.py | 22 +++++++++++++++++- rasa/cli/x.py | 2 +- rasa/core/constants.py | 4 ++++ rasa/core/run.py | 24 +++++++++++++++----- rasa/core/utils.py | 43 ++++++++++++++++++++++++++---------- rasa/shared/constants.py | 1 + rasa/utils/common.py | 27 +++++++++++++++++----- tests/cli/test_rasa_run.py | 7 ++++-- tests/cli/test_rasa_shell.py | 10 ++++++--- tests/cli/test_rasa_x.py | 9 +++++--- 10 files changed, 117 insertions(+), 32 deletions(-) diff --git a/rasa/cli/arguments/run.py b/rasa/cli/arguments/run.py index 8b9f7dfa893a..14bfcdc7ac91 100644 --- a/rasa/cli/arguments/run.py +++ b/rasa/cli/arguments/run.py @@ -55,6 +55,27 @@ def add_server_arguments(parser: argparse.ArgumentParser) -> None: default=None, help="Store logs in specified file.", ) + parser.add_argument( + "--use-syslog", action="store_true", help="Add syslog as a log handler", + ) + parser.add_argument( + "--syslog-address", + type=str, + default=constants.DEFAULT_SYSLOG_HOST, + help="Address of the syslog server. --use-sylog flag is required", + ) + parser.add_argument( + "--syslog-port", + type=int, + default=constants.DEFAULT_SYSLOG_PORT, + help="Port of the syslog server. --use-sylog flag is required", + ) + parser.add_argument( + "--syslog-protocol", + type=str, + default=constants.DEFAULT_PROTOCOL, + help="Protocol used with the syslog server. Can be UDP (default) or TCP ", + ) add_endpoint_param( parser, help_text="Configuration file for the model server and the connectors as a " @@ -113,7 +134,6 @@ def add_server_arguments(parser: argparse.ArgumentParser) -> None: help="If your ssl-keyfile is protected by a password, you can specify it " "using this paramer.", ) - channel_arguments = parser.add_argument_group("Channels") channel_arguments.add_argument( "--credentials", diff --git a/rasa/cli/x.py b/rasa/cli/x.py index 515c33910dcb..703ff4211dde 100644 --- a/rasa/cli/x.py +++ b/rasa/cli/x.py @@ -262,7 +262,7 @@ def _configure_logging(args: argparse.Namespace) -> None: configure_logging_and_warnings( log_level, warn_only_once=False, filter_repeated_logs=False ) - configure_file_logging(logging.root, args.log_file) + configure_file_logging(logging.root, args.log_file, False) logging.getLogger("werkzeug").setLevel(logging.WARNING) logging.getLogger("engineio").setLevel(logging.WARNING) diff --git a/rasa/core/constants.py b/rasa/core/constants.py index c3a9214370e2..28e4b5dc02b4 100644 --- a/rasa/core/constants.py +++ b/rasa/core/constants.py @@ -70,3 +70,7 @@ POLICY_PRIORITY = "priority" POLICY_FEATURIZER = "featurizer" POLICY_MAX_HISTORY = "max_history" + +DEFAULT_PROTOCOL = "UDP" +DEFAULT_SYSLOG_HOST = "localhost" +DEFAULT_SYSLOG_PORT = 514 diff --git a/rasa/core/run.py b/rasa/core/run.py index c0f4d5adeeed..768a79cb9641 100644 --- a/rasa/core/run.py +++ b/rasa/core/run.py @@ -29,7 +29,6 @@ def create_http_input_channels( channel: Optional[Text], credentials_file: Optional[Text] ) -> List["InputChannel"]: """Instantiate the chosen input channel.""" - if credentials_file: all_credentials = rasa.shared.utils.io.read_config_file(credentials_file) else: @@ -90,10 +89,15 @@ def configure_app( endpoints: Optional[AvailableEndpoints] = None, log_file: Optional[Text] = None, conversation_id: Optional[Text] = uuid.uuid4().hex, + use_syslog: bool = False, + syslog_address: Optional[Text] = None, + syslog_port: Optional[int] = None, + syslog_protocol: Optional[Text] = None, ) -> Sanic: """Run the agent.""" - - rasa.core.utils.configure_file_logging(logger, log_file) + rasa.core.utils.configure_file_logging( + logger, log_file, use_syslog, syslog_address, syslog_port, syslog_protocol, + ) if enable_api: app = server.create_app( @@ -160,9 +164,12 @@ def serve_application( ssl_ca_file: Optional[Text] = None, ssl_password: Optional[Text] = None, conversation_id: Optional[Text] = uuid.uuid4().hex, + use_syslog: Optional[bool] = False, + syslog_address: Optional[Text] = None, + syslog_port: Optional[int] = None, + syslog_protocol: Optional[Text] = None, ) -> None: """Run the API entrypoint.""" - if not channel and not credentials: channel = "cmdline" @@ -180,6 +187,10 @@ def serve_application( endpoints=endpoints, log_file=log_file, conversation_id=conversation_id, + use_syslog=use_syslog, + syslog_address=syslog_address, + syslog_port=syslog_port, + syslog_protocol=syslog_protocol, ) ssl_context = server.create_ssl_context( @@ -203,7 +214,10 @@ def serve_application( input_channels, endpoints, model_path, number_of_workers, enable_api ) - rasa.utils.common.update_sanic_log_level(log_file) + rasa.utils.common.update_sanic_log_level( + log_file, use_syslog, syslog_address, syslog_port, syslog_protocol, + ) + app.run( host=interface, port=port, diff --git a/rasa/core/utils.py b/rasa/core/utils.py index 95a940b4dcca..ae382c948c61 100644 --- a/rasa/core/utils.py +++ b/rasa/core/utils.py @@ -17,12 +17,13 @@ import rasa.shared.utils.io from rasa.constants import DEFAULT_SANIC_WORKERS, ENV_SANIC_WORKERS -from rasa.shared.constants import DEFAULT_ENDPOINTS_PATH +from rasa.shared.constants import DEFAULT_ENDPOINTS_PATH, TCP_PROTOCOL from rasa.core.lock_store import LockStore, RedisLockStore, InMemoryLockStore from rasa.utils.endpoints import EndpointConfig, read_endpoint_config from sanic import Sanic from sanic.views import CompositionView +from socket import SOCK_DGRAM, SOCK_STREAM import rasa.cli.utils as cli_utils @@ -30,24 +31,42 @@ def configure_file_logging( - logger_obj: logging.Logger, log_file: Optional[Text] + logger_obj: logging.Logger, + log_file: Optional[Text], + use_syslog: Optional[bool], + syslog_address: Optional[Text] = None, + syslog_port: Optional[int] = None, + syslog_protocol: Optional[Text] = None, ) -> None: """Configure logging to a file. Args: logger_obj: Logger object to configure. log_file: Path of log file to write to. + use_syslog: Add syslog as a logger. + syslog_address: Adress of the syslog server. + syslog_port: Port of the syslog server. + syslog_protocol: Protocol with the syslog server """ - if not log_file: - return - - formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") - file_handler = logging.FileHandler( - log_file, encoding=rasa.shared.utils.io.DEFAULT_ENCODING - ) - file_handler.setLevel(logger_obj.level) - file_handler.setFormatter(formatter) - logger_obj.addHandler(file_handler) + if use_syslog: + formatter = logging.Formatter( + "%(asctime)s [%(levelname)-5.5s] [%(process)d]" " %(message)s" + ) + socktype = SOCK_STREAM if syslog_protocol == TCP_PROTOCOL else SOCK_DGRAM + syslog_handler = logging.handlers.SysLogHandler( + address=(syslog_address, syslog_port), socktype=socktype, + ) + syslog_handler.setLevel(logger_obj.level) + syslog_handler.setFormatter(formatter) + logger_obj.addHandler(syslog_handler) + if log_file: + formatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") + file_handler = logging.FileHandler( + log_file, encoding=rasa.shared.utils.io.DEFAULT_ENCODING + ) + file_handler.setLevel(logger_obj.level) + file_handler.setFormatter(formatter) + logger_obj.addHandler(file_handler) def one_hot(hot_idx: int, length: int, dtype: Optional[Text] = None) -> np.ndarray: diff --git a/rasa/shared/constants.py b/rasa/shared/constants.py index eae0b2b1471b..fca75ac106ae 100644 --- a/rasa/shared/constants.py +++ b/rasa/shared/constants.py @@ -52,6 +52,7 @@ DEFAULT_LOG_LEVEL = "INFO" ENV_LOG_LEVEL = "LOG_LEVEL" +TCP_PROTOCOL = "TCP" DEFAULT_SENDER_ID = "default" UTTER_PREFIX = "utter_" diff --git a/rasa/utils/common.py b/rasa/utils/common.py index be58df410697..babf3f86996a 100644 --- a/rasa/utils/common.py +++ b/rasa/utils/common.py @@ -19,11 +19,11 @@ Set, ) +from socket import SOCK_DGRAM, SOCK_STREAM import numpy as np - import rasa.utils.io from rasa.constants import DEFAULT_LOG_LEVEL_LIBRARIES, ENV_LOG_LEVEL_LIBRARIES -from rasa.shared.constants import DEFAULT_LOG_LEVEL, ENV_LOG_LEVEL +from rasa.shared.constants import DEFAULT_LOG_LEVEL, ENV_LOG_LEVEL, TCP_PROTOCOL import rasa.shared.utils.io logger = logging.getLogger(__name__) @@ -178,9 +178,14 @@ def update_tensorflow_log_level() -> None: logging.getLogger("tensorflow").propagate = False -def update_sanic_log_level(log_file: Optional[Text] = None) -> None: - """Set the log level of sanic loggers to the log level specified in the environment - variable 'LOG_LEVEL_LIBRARIES'.""" +def update_sanic_log_level( + log_file: Optional[Text] = None, + use_syslog: Optional[bool] = False, + syslog_address: Optional[Text] = None, + syslog_port: Optional[int] = None, + syslog_protocol: Optional[Text] = None, +) -> None: + """Set the log level to 'LOG_LEVEL_LIBRARIES' environment variable .""" from sanic.log import logger, error_logger, access_logger log_level = os.environ.get(ENV_LOG_LEVEL_LIBRARIES, DEFAULT_LOG_LEVEL_LIBRARIES) @@ -201,6 +206,18 @@ def update_sanic_log_level(log_file: Optional[Text] = None) -> None: logger.addHandler(file_handler) error_logger.addHandler(file_handler) access_logger.addHandler(file_handler) + if use_syslog: + formatter = logging.Formatter( + "%(asctime)s [%(levelname)-5.5s] [%(process)d]" " %(message)s" + ) + socktype = SOCK_STREAM if syslog_protocol == TCP_PROTOCOL else SOCK_DGRAM + syslog_handler = logging.handlers.SysLogHandler( + address=(syslog_address, syslog_port), socktype=socktype, + ) + syslog_handler.setFormatter(formatter) + logger.addHandler(syslog_handler) + error_logger.addHandler(syslog_handler) + access_logger.addHandler(syslog_handler) def update_asyncio_log_level() -> None: diff --git a/tests/cli/test_rasa_run.py b/tests/cli/test_rasa_run.py index b8fcddb3d3ce..7cea40c9ca44 100644 --- a/tests/cli/test_rasa_run.py +++ b/tests/cli/test_rasa_run.py @@ -18,8 +18,11 @@ def test_run_help(run: Callable[..., RunResult]): output = run("run", "--help") help_text = """usage: rasa run [-h] [-v] [-vv] [--quiet] [-m MODEL] [--log-file LOG_FILE] - [--endpoints ENDPOINTS] [-i INTERFACE] [-p PORT] - [-t AUTH_TOKEN] [--cors [CORS [CORS ...]]] [--enable-api] + [--use-syslog] [--syslog-address SYSLOG_ADDRESS] + [--syslog-port SYSLOG_PORT] + [--syslog-protocol SYSLOG_PROTOCOL] [--endpoints ENDPOINTS] + [-i INTERFACE] [-p PORT] [-t AUTH_TOKEN] + [--cors [CORS [CORS ...]]] [--enable-api] [--response-timeout RESPONSE_TIMEOUT] [--remote-storage REMOTE_STORAGE] [--ssl-certificate SSL_CERTIFICATE] diff --git a/tests/cli/test_rasa_shell.py b/tests/cli/test_rasa_shell.py index 2d38887b12a9..8cd15c98649c 100644 --- a/tests/cli/test_rasa_shell.py +++ b/tests/cli/test_rasa_shell.py @@ -7,9 +7,13 @@ def test_shell_help(run: Callable[..., RunResult]): help_text = """usage: rasa shell [-h] [-v] [-vv] [--quiet] [--conversation-id CONVERSATION_ID] [-m MODEL] - [--log-file LOG_FILE] [--endpoints ENDPOINTS] [-i INTERFACE] - [-p PORT] [-t AUTH_TOKEN] [--cors [CORS [CORS ...]]] - [--enable-api] [--response-timeout RESPONSE_TIMEOUT] + [--log-file LOG_FILE] [--use-syslog] + [--syslog-address SYSLOG_ADDRESS] + [--syslog-port SYSLOG_PORT] + [--syslog-protocol SYSLOG_PROTOCOL] [--endpoints ENDPOINTS] + [-i INTERFACE] [-p PORT] [-t AUTH_TOKEN] + [--cors [CORS [CORS ...]]] [--enable-api] + [--response-timeout RESPONSE_TIMEOUT] [--remote-storage REMOTE_STORAGE] [--ssl-certificate SSL_CERTIFICATE] [--ssl-keyfile SSL_KEYFILE] [--ssl-ca-file SSL_CA_FILE] diff --git a/tests/cli/test_rasa_x.py b/tests/cli/test_rasa_x.py index c1435751a7b6..038c1da9197c 100644 --- a/tests/cli/test_rasa_x.py +++ b/tests/cli/test_rasa_x.py @@ -19,9 +19,12 @@ def test_x_help(run: Callable[..., RunResult]): help_text = """usage: rasa x [-h] [-v] [-vv] [--quiet] [-m MODEL] [--data DATA [DATA ...]] [-c CONFIG] [-d DOMAIN] [--no-prompt] [--production] [--rasa-x-port RASA_X_PORT] [--config-endpoint CONFIG_ENDPOINT] - [--log-file LOG_FILE] [--endpoints ENDPOINTS] [-i INTERFACE] - [-p PORT] [-t AUTH_TOKEN] [--cors [CORS [CORS ...]]] - [--enable-api] [--response-timeout RESPONSE_TIMEOUT] + [--log-file LOG_FILE] [--use-syslog] + [--syslog-address SYSLOG_ADDRESS] [--syslog-port SYSLOG_PORT] + [--syslog-protocol SYSLOG_PROTOCOL] [--endpoints ENDPOINTS] + [-i INTERFACE] [-p PORT] [-t AUTH_TOKEN] + [--cors [CORS [CORS ...]]] [--enable-api] + [--response-timeout RESPONSE_TIMEOUT] [--remote-storage REMOTE_STORAGE] [--ssl-certificate SSL_CERTIFICATE] [--ssl-keyfile SSL_KEYFILE] [--ssl-ca-file SSL_CA_FILE] [--ssl-password SSL_PASSWORD]