From 29c1cf7caa08a5e3209dfac7abe94f161be1d61f Mon Sep 17 00:00:00 2001 From: Kushal Das Date: Thu, 9 Jan 2020 14:47:28 +0530 Subject: [PATCH 1/2] Adds initial scripts to use rsyslog for logging in workstation `sd-rsyslog` is the output plugin of rsyslog to be installed in /usr/sbin `sdlog.conf` is the configuration of rsyslog in /etc/rsyslog.d/ `securedrop-redis-log` is part of the Qrexec service inside of sd-log vm. This will receive the messages from any other vm, and add them into a queue in Redis. `securedrop-log-saver` will be the service inside `sd-log` VM, this will monitor the queue, and save any incoming message to the `syslog.log` file of the respective directory for each VM. It also has the Makefile for the project. --- MANIFEST.in | 9 +- Makefile | 40 +++++++++ README.md | 43 +++++++-- VERSION | 1 + build-requirements.txt | 1 + changelog.md | 4 + requirements.in | 1 + requirements.txt | 9 ++ sd-rsyslog | 193 ++++++++++++++++++++++++++++++++++++++++ sd-rsyslog-example.conf | 4 + sdlog.conf | 4 + securedrop-log-saver | 54 +++++++++++ securedrop-log.service | 12 +++ securedrop-redis-log | 46 ++++++++++ securedrop.Log | 2 +- securedrop_log/VERSION | 1 - setup.py | 12 ++- update_version.sh | 10 +-- 18 files changed, 421 insertions(+), 25 deletions(-) create mode 100644 Makefile create mode 100644 VERSION create mode 100644 requirements.in create mode 100644 sd-rsyslog create mode 100644 sd-rsyslog-example.conf create mode 100644 sdlog.conf create mode 100755 securedrop-log-saver create mode 100644 securedrop-log.service create mode 100755 securedrop-redis-log delete mode 100644 securedrop_log/VERSION diff --git a/MANIFEST.in b/MANIFEST.in index 6d11dc9..9596bbc 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,8 +3,9 @@ include README.md include changelog.md include build-requirements.txt include requirements.txt -include securedrop_log/*.py -include securedrop_log/VERSION -include setup.py -include securedrop-log +include securedrop-log* +include securedrop-redis-log include securedrop.Log +include sd-rsyslog* +include sdlog.conf +include VERSION \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0a669f0 --- /dev/null +++ b/Makefile @@ -0,0 +1,40 @@ +DEFAULT_GOAL: help +SHELL := /bin/bash + +# Bandit is a static code analysis tool to detect security vulnerabilities in Python applications +# https://wiki.openstack.org/wiki/Security/Projects/Bandit +.PHONY: bandit +bandit: ## Run bandit with medium level excluding test-related folders + pip install --upgrade pip && \ + pip install --upgrade bandit!=1.6.0 && \ + bandit -ll --recursive . --exclude tests,.venv + +.PHONY: safety +safety: ## Runs `safety check` to check python dependencies for vulnerabilities + pip install --upgrade safety && \ + for req_file in `find . -type f -name '*requirements.txt'`; do \ + echo "Checking file $$req_file" \ + && safety check --full-report -r $$req_file \ + && echo -e '\n' \ + || exit 1; \ + done + +.PHONY: update-pip-requirements +update-pip-requirements: ## Updates all Python requirements files via pip-compile. + pip-compile --generate-hashes --output-file requirements.txt requirements.in + + +# Explaination of the below shell command should it ever break. +# 1. Set the field separator to ": ##" and any make targets that might appear between : and ## +# 2. Use sed-like syntax to remove the make targets +# 3. Format the split fields into $$1) the target name (in blue) and $$2) the target descrption +# 4. Pass this file as an arg to awk +# 5. Sort it alphabetically +# 6. Format columns with colon as delimiter. +.PHONY: help +help: ## Print this message and exit. + @printf "Makefile for developing and testing the SecureDrop Logging system.\n" + @printf "Subcommands:\n\n" + @awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \ + | sort \ + | column -s ':' -t diff --git a/README.md b/README.md index efaf8e8..0379fca 100644 --- a/README.md +++ b/README.md @@ -24,19 +24,47 @@ Add the following content to `/etc/qubes-rpc/securedrop.Log` /usr/sbin/securedrop-log ``` -and then place `securedrop-log` script to `/usr/sbin/` directory and make sure that -it is executable. +and then place `securedrop-redis-log` and `securedrop-log-saver` scripts to the +virtualenv at `/opt/venvs/securedrop-log` and create links to `/usr/sbin/` +directory and make sure that they are executable. This step will be automated via +the Debian package. + + +Copy `securedrop-log.service` file to `/usr/systemd/system` and then + +``` +sudo systemctl daemon-reload +sudo systemctl start redis +sudo systemctl start securedrop-log +``` + +To test the logging, make sure to execute `securedrop-log-saver` from a terminal in `sd-log` +and check the ~/QubesIncomingLogs/vmname/syslog.log file via **tail -f**. + ### To use from any Python code in workvm +Put `sd-rsyslog-example.conf` file to `/etc/sd-rsyslog.conf`, make sure update +it so that is shows the right **localvm** name. + +Copy `sd-rsyslog` executable to **/usr/sbin**, and remember to `chmod +x` +the binary. + +Next, restart the rsyslog service. + +``` +systemctl restart rsyslog +``` + + Here is an example code using Python logging ```Python import logging -from securedrop_log import SecureDropLog +import logging.handlers def main(): - handler = SecureDropLog("workvm", "proxy-debian") + handler = logging.handlers.SysLogHandler(address="/dev/log") logging.basicConfig(level=logging.DEBUG, handlers=[handler]) logger = logging.getLogger("example") @@ -48,8 +76,9 @@ if __name__ == "__main__": ``` -## The journalctl example +Or use the logger command. -You will need `python3-systemd` package for the same. +``` +logger This line should show in the syslog.log file in the sd-log file. +``` -The code is in `journal-example.py` file. \ No newline at end of file diff --git a/VERSION b/VERSION new file mode 100644 index 0000000..81340c7 --- /dev/null +++ b/VERSION @@ -0,0 +1 @@ +0.0.4 diff --git a/build-requirements.txt b/build-requirements.txt index e69de29..af566a3 100644 --- a/build-requirements.txt +++ b/build-requirements.txt @@ -0,0 +1 @@ +redis==3.3.11 --hash=sha256:022f124431ae16ee3a3a69c8016e3e2b057b4f4e0bfa7787b6271d893890c3cc diff --git a/changelog.md b/changelog.md index b82e78e..c9fba47 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,9 @@ # Changelog +## 0.0.4 + + * Converts into rsyslog based logging system. + ## 0.0.3 * Fixes typos MANIFEST.in and setup.py diff --git a/requirements.in b/requirements.in new file mode 100644 index 0000000..767bdac --- /dev/null +++ b/requirements.in @@ -0,0 +1 @@ +redis==3.3.11 \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index e69de29..3c896d5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -0,0 +1,9 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# pip-compile --generate-hashes --output-file=requirements.txt requirements.in +# +redis==3.3.11 \ + --hash=sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62 \ + --hash=sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2 diff --git a/sd-rsyslog b/sd-rsyslog new file mode 100644 index 0000000..4f3233e --- /dev/null +++ b/sd-rsyslog @@ -0,0 +1,193 @@ +#!/opt/venvs/securedrop-log/bin/python3 +"""A skeleton for a Python rsyslog output plugin with error handling. +Requires Python 3. + +To integrate a plugin based on this skeleton with rsyslog, configure an +'omprog' action like the following: + action(type="omprog" + binary="/usr/bin/myplugin.py" + output="/var/log/myplugin.log" + confirmMessages="on" + ...) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + -or- + see COPYING.ASL20 in the source distribution + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +""" + +import sys +import os +import logging +import configparser +from subprocess import Popen, PIPE + +# Global definitions specific to your plugin +process = None + +class RecoverableError(Exception): + """An error that has caused the processing of the current message to + fail, but does not require restarting the plugin. + + An example of such an error would be a temporary loss of connection to + a database or a server. If such an error occurs in the onMessage function, + your plugin should wrap it in a RecoverableError before raising it. + For example: + + try: + # code that connects to a database + except DbConnectionError as e: + raise RecoverableError from e + + Recoverable errors will cause the 'omprog' action to be temporarily + suspended by rsyslog, during a period that can be configured using the + "action.resumeInterval" action parameter. When the action is resumed, + rsyslog will resend the failed message to your plugin. + """ + + +def onInit(): + """Do everything that is needed to initialize processing (e.g. open files, + create handles, connect to systems...). + """ + # Apart from processing the logs received from rsyslog, you want your plugin + # to be able to report its own logs in some way. This will facilitate + # diagnosing problems and debugging your code. Here we set up the standard + # Python logging system to output the logs to stderr. In the rsyslog + # configuration, you can configure the 'omprog' action to capture the stderr + # of your plugin by specifying the action's "output" parameter. + logging.basicConfig(stream=sys.stderr, + level=logging.WARNING, + format='%(asctime)s %(levelname)s %(message)s') + + # This is an example of a debug log. (Note that for debug logs to be + # emitted you must set 'level' to logging.DEBUG above.) + logging.debug("onInit called") + + + global process + if not os.path.exists("/etc/sd-rsyslog.conf"): + print("Please create the configuration file at /etc/sd-rsyslog.conf", file=sys.stderr) + sys.exit(1) + config = configparser.ConfigParser() + config.read('/etc/sd-rsyslog.conf') + logvmname = config['sd-rsyslog']['remotevm'] + localvmname = config['sd-rsyslog']['localvm'] + process = Popen( + ["/usr/lib/qubes/qrexec-client-vm", logvmname, "securedrop.Log"], + stdin=PIPE, + stdout=PIPE, + stderr=PIPE, + ) + process.stdin.write(localvmname.encode("utf-8")) + process.stdin.write(b"\n") + process.stdin.flush() + + +def onMessage(msg): + """Process one log message received from rsyslog (e.g. send it to a + database). If this function raises an error, the message will be retried + by rsyslog. + + Args: + msg (str): the log message. Does NOT include a trailing newline. + + Raises: + RecoverableError: If a recoverable error occurs. The message will be + retried without restarting the plugin. + Exception: If a non-recoverable error occurs. The plugin will be + restarted before retrying the message. + """ + logging.debug("onMessage called") + + # For illustrative purposes, this plugin skeleton appends the received logs + # to a file. When implementing your plugin, remove the following code. + global process + process.stdin.write(msg.encode("utf-8")) + process.stdin.write(b"\n") + process.stdin.flush() + + +def onExit(): + """Do everything that is needed to finish processing (e.g. close files, + handles, disconnect from systems...). This is being called immediately + before exiting. + + This function should not raise any error. If it does, the error will be + logged as a warning and ignored. + """ + logging.debug("onExit called") + + # For illustrative purposes, this plugin skeleton appends the received logs + # to a file. When implementing your plugin, remove the following code. + global process + process.stdin.flush() + + +""" +------------------------------------------------------- +This is plumbing that DOES NOT need to be CHANGED +------------------------------------------------------- +This is the main loop that receives messages from rsyslog via stdin, +invokes the above entrypoints, and provides status codes to rsyslog +via stdout. In most cases, modifying this code should not be necessary. +""" +try: + onInit() +except Exception as e: + # If an error occurs during initialization, log it and terminate. The + # 'omprog' action will eventually restart the program. + logging.exception("Initialization error, exiting program") + sys.exit(1) + +# Tell rsyslog we are ready to start processing messages: +print("OK", flush=True) + +endedWithError = False +try: + line = sys.stdin.readline() + while line: + line = line.rstrip('\n') + try: + onMessage(line) + status = "OK" + except RecoverableError as e: + # Any line written to stdout that is not a status code will be + # treated as a recoverable error by 'omprog', and cause the action + # to be temporarily suspended. In this skeleton, we simply return + # a one-line representation of the Python exception. (If debugging + # is enabled in rsyslog, this line will appear in the debug logs.) + status = repr(e) + # We also log the complete exception to stderr (or to the logging + # handler(s) configured in doInit, if any). + logging.exception(e) + + # Send the status code (or the one-line error message) to rsyslog: + print(status, flush=True) + line = sys.stdin.readline() + +except Exception: + # If a non-recoverable error occurs, log it and terminate. The 'omprog' + # action will eventually restart the program. + logging.exception("Unrecoverable error, exiting program") + endedWithError = True + +try: + onExit() +except Exception: + logging.warning("Exception ignored in onExit", exc_info=True) + +if endedWithError: + sys.exit(1) +else: + sys.exit(0) + diff --git a/sd-rsyslog-example.conf b/sd-rsyslog-example.conf new file mode 100644 index 0000000..b79af60 --- /dev/null +++ b/sd-rsyslog-example.conf @@ -0,0 +1,4 @@ +[sd-rsyslog] +remotevm = sd-log +localvm = sd-app + diff --git a/sdlog.conf b/sdlog.conf new file mode 100644 index 0000000..be8cac7 --- /dev/null +++ b/sdlog.conf @@ -0,0 +1,4 @@ +module(load="omprog") +action(type="omprog" + binary="/usr/sbin/sd-rsyslog" + template="RSYSLOG_TraditionalFileFormat") diff --git a/securedrop-log-saver b/securedrop-log-saver new file mode 100755 index 0000000..3de1c76 --- /dev/null +++ b/securedrop-log-saver @@ -0,0 +1,54 @@ +#!/opt/venvs/securedrop-log/bin/python3 + +import os +import sys +import redis +import errno + + +def main(): + rclient = redis.Redis() + # This is the cache of open files for each vm + openfiles = {} + try: + while True: + # Wait for the next message + qname, data = rclient.blpop("syslogmsg") + msg = data.decode("utf-8") + vmname, msg_str = msg.split("::", 1) + + if vmname in openfiles: + fh = openfiles[vmname] + else: + # First open a file + filepath = os.path.join( + os.getenv("HOME", "/"), + "QubesIncomingLogs", + f"{vmname}", + "syslog.log", + ) + dirpath = os.path.dirname(filepath) + try: + os.makedirs(dirpath) + except OSError as err: + if err.errno != errno.EEXIST: + raise + fh = open(filepath, "a") + + # cache it for the next call + openfiles[vmname] = fh + + # Now just write and flush + fh.write(msg_str) + fh.write("\n") + fh.flush() + except Exception as e: + print(e, file=sys.stderr) + # Clean up all open files + for k, v in openfiles: + v.close() + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/securedrop-log.service b/securedrop-log.service new file mode 100644 index 0000000..796df4e --- /dev/null +++ b/securedrop-log.service @@ -0,0 +1,12 @@ +[Unit] +Description=securedrop logging Service +After=network.target + +[Service] +Type=simple +User=user +ExecStart=/usr/sbin/securedrop-log-saver +Restart=on-abort + +[Install] +WantedBy=multi-user.target diff --git a/securedrop-redis-log b/securedrop-redis-log new file mode 100755 index 0000000..740c2f8 --- /dev/null +++ b/securedrop-redis-log @@ -0,0 +1,46 @@ +#!/opt/venvs/securedrop-log/bin/python3 + +from __future__ import print_function + +import tempfile +import io +import sys +import os +import errno +import shutil +import subprocess +import redis +from datetime import datetime + + +def sanitize_line(untrusted_line): + line = bytearray(untrusted_line) + for i, c in enumerate(line): + if c >= 0x20 and c <= 0x7E: + pass + else: + line[i] = 0x2E + return bytearray(line).decode("ascii") + + +stdin = sys.stdin.buffer # python3 + + +rd = redis.Redis() + + +def log(msg, vmname="remote"): + global rd + redis_msg = f"{vmname}::{msg}" + rd.rpush("syslogmsg", redis_msg) + + +# the first line is always the remote vm name +untrusted_line = stdin.readline() +qrexec_remote = untrusted_line.rstrip(b"\n").decode("utf-8") +while True: + untrusted_line = stdin.readline() + if untrusted_line == b"": + break + + log(sanitize_line(untrusted_line.rstrip(b"\n")), qrexec_remote) diff --git a/securedrop.Log b/securedrop.Log index 0ebaf24..bfb53e4 100644 --- a/securedrop.Log +++ b/securedrop.Log @@ -1 +1 @@ -/usr/sbin/securedrop-log +/usr/sbin/securedrop-redis-log diff --git a/securedrop_log/VERSION b/securedrop_log/VERSION deleted file mode 100644 index bcab45a..0000000 --- a/securedrop_log/VERSION +++ /dev/null @@ -1 +0,0 @@ -0.0.3 diff --git a/setup.py b/setup.py index 790b992..efbe8f2 100644 --- a/setup.py +++ b/setup.py @@ -4,8 +4,8 @@ with open("README.md", "r") as fh: long_description = fh.read() -version = pkgutil.get_data("securedrop_log", "VERSION").decode("utf-8") -version = version.strip() +with open("VERSION") as fh: + version = fh.read().strip() setuptools.setup( @@ -20,15 +20,13 @@ install_requires=[], python_requires=">=3.5", url="https://github.com/freedomofpress/securedrop-log", - packages=["securedrop_log",], - package_data={"securedrop_log": ["VERSION"],}, - classifiers=( + classifiers=[ "Development Status :: 3 - Alpha", "Programming Language :: Python :: 3", "Topic :: Software Development :: Libraries :: Python Modules", "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)", "Intended Audience :: Developers", "Operating System :: POSIX :: Linux", - ), - data_files=[("sbin", ["securedrop-log"])], + ], + data_files=[("sbin", ["securedrop-log", "securedrop-log-saver", "securedrop-redis-log"])], ) diff --git a/update_version.sh b/update_version.sh index 18c1330..3bd6903 100755 --- a/update_version.sh +++ b/update_version.sh @@ -10,17 +10,17 @@ if [ -z "$NEW_VERSION" ]; then exit 1 fi -# Get the old version from securedrop_log/VERSION -OLD_VERSION=$(cat securedrop_log/VERSION) +# Get the old version from VERSION +OLD_VERSION=$(cat VERSION) if [ -z "$OLD_VERSION" ]; then echo "Couldn't find the old version: does this script need to be updated?" exit 1 fi -# Update the version in securedrop_log/VERSION (setup.py is done automatically) +# Update the version in VERSION (setup.py is done automatically) if [[ "$OSTYPE" == "darwin"* ]]; then # The empty '' after sed -i is required on macOS to indicate no backup file should be saved. - sed -i '' "s@$(echo "${OLD_VERSION}" | sed 's/\./\\./g')@$NEW_VERSION@g" securedrop_log/VERSION + sed -i '' "s@$(echo "${OLD_VERSION}" | sed 's/\./\\./g')@$NEW_VERSION@g" VERSION else - sed -i "s@$(echo "${OLD_VERSION}" | sed 's/\./\\./g')@$NEW_VERSION@g" securedrop_log/VERSION + sed -i "s@$(echo "${OLD_VERSION}" | sed 's/\./\\./g')@$NEW_VERSION@g" VERSION fi From 9fbcae497efa1076990ad6f5fcd550e44a2eedf1 Mon Sep 17 00:00:00 2001 From: mickael e Date: Thu, 6 Feb 2020 11:15:55 -0500 Subject: [PATCH 2/2] ci: Add build-buster job --- .circleci/config.yml | 50 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index dd6ad37..71dfd2d 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,3 +1,42 @@ +--- +common-steps: + - &install_packaging_dependencies + run: + name: Install Debian packaging dependencies and download wheels + command: | + mkdir ~/packaging && cd ~/packaging + git config --global --unset url.ssh://git@github.com.insteadof + git clone https://github.com/freedomofpress/securedrop-debian-packaging.git + cd securedrop-debian-packaging + make install-deps + PKG_DIR=~/project make requirements + + - &verify_requirements + run: + name: Ensure that build-requirements.txt and requirements.txt are in sync. + command: | + cd ~/project + # Return 1 if unstaged changes exist (after `make requirements` in the + # previous run step), else return 0. + git diff --quiet + + - &make_source_tarball + run: + name: Tag and make source tarball + command: | + cd ~/project + ./update_version.sh 1000.0 # Dummy version number, doesn't matter what we put here + python3 setup.py sdist + + - &build_debian_package + run: + name: Build debian package + command: | + cd ~/packaging/securedrop-debian-packaging + export PKG_VERSION=1000.0 + export PKG_PATH=/home/circleci/project/dist/securedrop-log-$PKG_VERSION.tar.gz + make securedrop-log + version: 2 jobs: test: @@ -9,8 +48,19 @@ jobs: name: Run tests command: python3 -m unittest + build-buster: + docker: + - image: circleci/python:3.7-buster + steps: + - checkout + - *install_packaging_dependencies + - *verify_requirements + - *make_source_tarball + - *build_debian_package + workflows: version: 2 per_pr: jobs: - test + - build-buster