Merge pull request #8 from freedomofpress/oh_my_syslog

Adds initial scripts to use rsyslog for logging in workstation

.circleci/config.yml

@@ -1,3 +1,42 @@
+---
+common-steps:
+  - &install_packaging_dependencies
+    run:
+      name: Install Debian packaging dependencies and download wheels
+      command: |
+        mkdir ~/packaging && cd ~/packaging
+        git config --global --unset url.ssh://[email protected]
+        git clone https://github.com/freedomofpress/securedrop-debian-packaging.git
+        cd securedrop-debian-packaging
+        make install-deps
+        PKG_DIR=~/project make requirements
+
+  - &verify_requirements
+    run:
+      name: Ensure that build-requirements.txt and requirements.txt are in sync.
+      command: |
+        cd ~/project
+        # Return 1 if unstaged changes exist (after `make requirements` in the
+        # previous run step), else return 0.
+        git diff --quiet
+
+  - &make_source_tarball
+    run:
+      name: Tag and make source tarball
+      command: |
+        cd ~/project
+        ./update_version.sh 1000.0  # Dummy version number, doesn't matter what we put here
+        python3 setup.py sdist
+
+  - &build_debian_package
+    run:
+      name: Build debian package
+      command: |
+        cd ~/packaging/securedrop-debian-packaging
+        export PKG_VERSION=1000.0
+        export PKG_PATH=/home/circleci/project/dist/securedrop-log-$PKG_VERSION.tar.gz
+        make securedrop-log
+
 version: 2
 jobs:
   test:
@@ -9,8 +48,19 @@ jobs:
           name: Run tests
           command: python3 -m unittest
 
+  build-buster:
+    docker:
+      - image: circleci/python:3.7-buster
+    steps:
+      - checkout
+      - *install_packaging_dependencies
+      - *verify_requirements
+      - *make_source_tarball
+      - *build_debian_package
+
 workflows:
   version: 2
   per_pr:
     jobs:
       - test
+      - build-buster

MANIFEST.in

@@ -3,8 +3,9 @@ include README.md
 include changelog.md
 include build-requirements.txt
 include requirements.txt
-include securedrop_log/*.py
-include securedrop_log/VERSION
-include setup.py
-include securedrop-log
+include securedrop-log*
+include securedrop-redis-log
 include securedrop.Log
+include sd-rsyslog*
+include sdlog.conf
+include VERSION

Makefile

@@ -0,0 +1,40 @@
+DEFAULT_GOAL: help
+SHELL := /bin/bash
+
+# Bandit is a static code analysis tool to detect security vulnerabilities in Python applications
+# https://wiki.openstack.org/wiki/Security/Projects/Bandit
+.PHONY: bandit
+bandit: ## Run bandit with medium level excluding test-related folders
+	pip install --upgrade pip && \
+        pip install --upgrade bandit!=1.6.0 && \
+	bandit -ll --recursive . --exclude tests,.venv
+
+.PHONY: safety
+safety: ## Runs `safety check` to check python dependencies for vulnerabilities
+	pip install --upgrade safety && \
+		for req_file in `find . -type f -name '*requirements.txt'`; do \
+			echo "Checking file $$req_file" \
+			&& safety check --full-report -r $$req_file \
+			&& echo -e '\n' \
+			|| exit 1; \
+		done
+
+.PHONY: update-pip-requirements
+update-pip-requirements: ## Updates all Python requirements files via pip-compile.
+	pip-compile --generate-hashes --output-file requirements.txt requirements.in
+
+
+# Explaination of the below shell command should it ever break.
+# 1. Set the field separator to ": ##" and any make targets that might appear between : and ##
+# 2. Use sed-like syntax to remove the make targets
+# 3. Format the split fields into $$1) the target name (in blue) and $$2) the target descrption
+# 4. Pass this file as an arg to awk
+# 5. Sort it alphabetically
+# 6. Format columns with colon as delimiter.
+.PHONY: help
+help: ## Print this message and exit.
+	@printf "Makefile for developing and testing the SecureDrop Logging system.\n"
+	@printf "Subcommands:\n\n"
+	@awk 'BEGIN {FS = ":.*?## "} /^[0-9a-zA-Z_-]+:.*?## / {printf "\033[36m%s\033[0m : %s\n", $$1, $$2}' $(MAKEFILE_LIST) \
+		| sort \
+		| column -s ':' -t

README.md

@@ -24,19 +24,47 @@ Add the following content to `/etc/qubes-rpc/securedrop.Log`
 /usr/sbin/securedrop-log
 ```
 
-and then place `securedrop-log` script to `/usr/sbin/` directory and make sure that
-it is executable.
+and then place `securedrop-redis-log` and `securedrop-log-saver` scripts to the
+virtualenv at `/opt/venvs/securedrop-log` and create links to `/usr/sbin/`
+directory and make sure that they are executable. This step will be automated via
+the Debian package.
+
+
+Copy `securedrop-log.service` file to `/usr/systemd/system` and then
+
+```
+sudo systemctl daemon-reload
+sudo systemctl start redis
+sudo systemctl start securedrop-log
+```
+
+To test the logging, make sure to execute `securedrop-log-saver` from a terminal in `sd-log`
+and check the ~/QubesIncomingLogs/vmname/syslog.log file via **tail -f**.
+
 
 ### To use from any Python code in workvm
 
+Put `sd-rsyslog-example.conf` file to `/etc/sd-rsyslog.conf`, make sure update
+it so that is shows the right **localvm** name.
+
+Copy `sd-rsyslog` executable to **/usr/sbin**, and remember to `chmod +x`
+the binary. 
+
+Next, restart the rsyslog service.
+
+```
+systemctl restart rsyslog
+```
+
+
 Here is an example code using Python logging
 
 ```Python
 import logging
-from securedrop_log import SecureDropLog
+import logging.handlers
 
 def main():
-    handler = SecureDropLog("workvm", "proxy-debian")
+    handler = logging.handlers.SysLogHandler(address="/dev/log")
     logging.basicConfig(level=logging.DEBUG, handlers=[handler])
     logger = logging.getLogger("example")
 
@@ -48,8 +76,9 @@ if __name__ == "__main__":
 
 ```
 
-## The journalctl example
+Or use the logger command.
 
-You will need `python3-systemd` package for the same.
+```
+logger This line should show in the syslog.log file in the sd-log file.
+```
 
-The code is in `journal-example.py` file.

VERSION

@@ -0,0 +1 @@
+0.0.4

build-requirements.txt

@@ -0,0 +1 @@
+redis==3.3.11 --hash=sha256:022f124431ae16ee3a3a69c8016e3e2b057b4f4e0bfa7787b6271d893890c3cc

changelog.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## 0.0.4
+
+  * Converts into rsyslog based logging system.
+
 ## 0.0.3
 
   * Fixes typos MANIFEST.in and setup.py

requirements.in

@@ -0,0 +1 @@
+redis==3.3.11

requirements.txt

@@ -0,0 +1,9 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    pip-compile --generate-hashes --output-file=requirements.txt requirements.in
+#
+redis==3.3.11 \
+    --hash=sha256:3613daad9ce5951e426f460deddd5caf469e08a3af633e9578fc77d362becf62 \
+    --hash=sha256:8d0fc278d3f5e1249967cba2eb4a5632d19e45ce5c09442b8422d15ee2c22cc2