From 47ce0d346e2c529292add6f850d40c92529e3779 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sat, 27 Jan 2018 20:31:12 +0100 Subject: [PATCH 01/19] staging: set smtp_replay to a fake domain So we don't have to worry about sending too many failed attempts to a real SMTP server. --- install_files/ansible-base/group_vars/staging.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install_files/ansible-base/group_vars/staging.yml b/install_files/ansible-base/group_vars/staging.yml index f2a1b0da6b..9c4f9ee672 100644 --- a/install_files/ansible-base/group_vars/staging.yml +++ b/install_files/ansible-base/group_vars/staging.yml @@ -34,7 +34,7 @@ apache_logging_level: "info" ossec_alert_gpg_public_key: "test_admin_key.pub" ossec_gpg_fpr: "53E1113AC1F25027BA5D475B1141E2BBB5E53711" ossec_alert_email: "ossec@ossec.test" -smtp_relay: "smtp.gmail.com" +smtp_relay: "smtp.faketld" smtp_relay_port: "587" sasl_username: "test" sasl_domain: "ossec.test" From 2fc3095e62574569e774b9ab3d7020da125b0627 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sat, 27 Jan 2018 20:33:26 +0100 Subject: [PATCH 02/19] lint: ignore emacs backup files --- .dockerignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.dockerignore b/.dockerignore index eb6ab1f6bf..6243e836a3 100644 --- a/.dockerignore +++ b/.dockerignore @@ -7,6 +7,9 @@ **/venv **/.venv +**/*~ +**/#* + **/__pycache__ **/*.pyc securedrop/.sass-cache From f83e647793c9fc981b6ccf0de677c0ecb394921b Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sat, 27 Jan 2018 20:43:59 +0100 Subject: [PATCH 03/19] manage: use args.store_dir instead of config.STORE_DIR --- securedrop/manage.py | 9 ++++++--- securedrop/tests/test_manage.py | 3 ++- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/securedrop/manage.py b/securedrop/manage.py index a3e801d71c..45be73ce56 100755 --- a/securedrop/manage.py +++ b/securedrop/manage.py @@ -51,15 +51,15 @@ def reset(args): # Clear submission/reply storage try: - os.stat(config.STORE_DIR) + os.stat(args.store_dir) except OSError: pass else: - for source_dir in os.listdir(config.STORE_DIR): + for source_dir in os.listdir(args.store_dir): try: # Each entry in STORE_DIR is a directory corresponding # to a source - shutil.rmtree(os.path.join(config.STORE_DIR, source_dir)) + shutil.rmtree(os.path.join(args.store_dir, source_dir)) except OSError: pass return 0 @@ -254,6 +254,9 @@ def get_args(): parser = argparse.ArgumentParser(prog=__file__, description='Management ' 'and testing utility for SecureDrop.') parser.add_argument('-v', '--verbose', action='store_true') + parser.add_argument('--store-dir', + default=config.STORE_DIR, + help=('directory in which the documents are stored')) subps = parser.add_subparsers() # Run WSGI app run_subp = subps.add_parser('run', help='Run the Werkzeug source & ' diff --git a/securedrop/tests/test_manage.py b/securedrop/tests/test_manage.py index 910c79ec1c..69cefde76e 100644 --- a/securedrop/tests/test_manage.py +++ b/securedrop/tests/test_manage.py @@ -170,7 +170,8 @@ def test_reset(journalist_app, test_journo, config): # We need to override the config to point at the per-test DB manage.config = config - return_value = manage.reset(args=None) + args = argparse.Namespace(store_dir=config.STORE_DIR) + return_value = manage.reset(args=args) assert return_value == 0 assert os.path.exists(config.DATABASE_FILE) assert os.path.exists(config.STORE_DIR) From 5a43ac0a81f871c534ded6e360c663fc217b7e57 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sat, 27 Jan 2018 20:49:48 +0100 Subject: [PATCH 04/19] manage: implement the how_many_submissions_today sub-command --- securedrop/manage.py | 41 +++++++++++++++++++++++++++++++-- securedrop/tests/test_manage.py | 29 ++++++++++++++++++++++- 2 files changed, 67 insertions(+), 3 deletions(-) diff --git a/securedrop/manage.py b/securedrop/manage.py index 45be73ce56..79a80fb5f2 100755 --- a/securedrop/manage.py +++ b/securedrop/manage.py @@ -3,6 +3,7 @@ import argparse import codecs +import datetime import logging import os import pwd @@ -14,15 +15,16 @@ import traceback from flask import current_app -from sqlalchemy import text +from sqlalchemy import text, create_engine from sqlalchemy.orm.exc import NoResultFound +from sqlalchemy.orm import sessionmaker os.environ['SECUREDROP_ENV'] = 'dev' # noqa from sdconfig import config import journalist_app from db import db -from models import Journalist, PasswordError, InvalidUsernameException +from models import Source, Journalist, PasswordError, InvalidUsernameException from management.run import run logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s') @@ -250,10 +252,35 @@ def init_db(args): os.chown('/var/lib/securedrop/db.sqlite', user.pw_uid, user.pw_gid) +def were_there_submissions_today(args): + if config.DATABASE_ENGINE == "sqlite": + db_uri = (config.DATABASE_ENGINE + ":///" + + config.DATABASE_FILE) + else: + db_uri = ( + config.DATABASE_ENGINE + '://' + + config.DATABASE_USERNAME + ':' + + config.DATABASE_PASSWORD + '@' + + config.DATABASE_HOST + '/' + + config.DATABASE_NAME + ) + session = sessionmaker(bind=create_engine(db_uri))() + something = session.query(Source).filter( + Source.last_updated > + datetime.datetime.utcnow() - datetime.timedelta(hours=24) + ).count() > 0 + count_file = os.path.join(args.data_root, 'submissions_today.txt') + open(count_file, 'w').write(something and '1' or '0') + + def get_args(): parser = argparse.ArgumentParser(prog=__file__, description='Management ' 'and testing utility for SecureDrop.') parser.add_argument('-v', '--verbose', action='store_true') + parser.add_argument('--data-root', + default=config.SECUREDROP_DATA_ROOT, + help=('directory in which the securedrop ' + 'data is stored')) parser.add_argument('--store-dir', default=config.STORE_DIR, help=('directory in which the documents are stored')) @@ -289,6 +316,8 @@ def get_args(): set_clean_tmp_parser(subps, 'clean-tmp') set_clean_tmp_parser(subps, 'clean_tmp') + set_were_there_submissions_today(subps) + init_db_subp = subps.add_parser('init-db', help='initialize the DB') init_db_subp.add_argument('-u', '--user', help='Unix user for the DB', @@ -298,6 +327,14 @@ def get_args(): return parser +def set_were_there_submissions_today(subps): + parser = subps.add_parser( + 'were-there-submissions-today', + help=('Update the file indicating ' + 'iff submissions were received in the past 24h')) + parser.set_defaults(func=were_there_submissions_today) + + def set_clean_tmp_parser(subps, name): parser = subps.add_parser(name, help='Cleanup the ' 'SecureDrop temp directory.') diff --git a/securedrop/tests/test_manage.py b/securedrop/tests/test_manage.py index 69cefde76e..21d3b1ebc5 100644 --- a/securedrop/tests/test_manage.py +++ b/securedrop/tests/test_manage.py @@ -2,6 +2,7 @@ import argparse import io +import datetime import logging import os import manage @@ -13,7 +14,8 @@ os.environ['SECUREDROP_ENV'] = 'test' # noqa -from models import Journalist +from models import Journalist, db +from utils import db_helper YUBIKEY_HOTP = ['cb a0 5f ad 41 a2 ff 4e eb 53 56 3a 1b f7 23 2e ce fc dc', @@ -222,3 +224,28 @@ def test_clean_tmp_removed(config, caplog): manage.setup_verbosity(args) manage.clean_tmp(args) assert 'FILE removed' in caplog.text + + +def test_were_there_submissions_today(source_app, config): + original_config = manage.config + try: + # We need to override the config to point at the per-test DB + manage.config = config + data_root = config.SECUREDROP_DATA_ROOT + args = argparse.Namespace(data_root=data_root, + verbose=logging.DEBUG) + + with source_app.app_context(): + count_file = os.path.join(data_root, 'submissions_today.txt') + source, codename = db_helper.init_source_without_keypair() + source.last_updated = (datetime.datetime.utcnow() - + datetime.timedelta(hours=24*2)) + db.session.commit() + manage.were_there_submissions_today(args) + assert io.open(count_file).read() == "0" + source.last_updated = datetime.datetime.utcnow() + db.session.commit() + manage.were_there_submissions_today(args) + assert io.open(count_file).read() == "1" + finally: + manage.config = original_config From c7cc8421da3df8a9f78a37e852ca650b34e89651 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sat, 27 Jan 2018 22:21:56 +0100 Subject: [PATCH 05/19] testinfra: define staging as a multi-machine context --- .../ansible-base/group_vars/staging.yml | 3 ++ molecule/aws/scripts/ci-tester.sh | 1 + testinfra/test.py | 30 +++++++++++++++++++ 3 files changed, 34 insertions(+) diff --git a/install_files/ansible-base/group_vars/staging.yml b/install_files/ansible-base/group_vars/staging.yml index 9c4f9ee672..c27e5a9619 100644 --- a/install_files/ansible-base/group_vars/staging.yml +++ b/install_files/ansible-base/group_vars/staging.yml @@ -34,6 +34,9 @@ apache_logging_level: "info" ossec_alert_gpg_public_key: "test_admin_key.pub" ossec_gpg_fpr: "53E1113AC1F25027BA5D475B1141E2BBB5E53711" ossec_alert_email: "ossec@ossec.test" +journalist_alert_gpg_public_key: "test_admin_key.pub" +journalist_gpg_fpr: "53E1113AC1F25027BA5D475B1141E2BBB5E53711" +journalist_alert_email: "journalist@ossec.test" smtp_relay: "smtp.faketld" smtp_relay_port: "587" sasl_username: "test" diff --git a/molecule/aws/scripts/ci-tester.sh b/molecule/aws/scripts/ci-tester.sh index 9c6fe0d00c..31ba92b112 100755 --- a/molecule/aws/scripts/ci-tester.sh +++ b/molecule/aws/scripts/ci-tester.sh @@ -7,6 +7,7 @@ if [ "$?" == "0" ]; then "staging") ./testinfra/test.py "app-$CI_SD_ENV" || export TEST_FAIL=true ./testinfra/test.py "mon-$CI_SD_ENV" || export TEST_FAIL=true + ./testinfra/test.py staging || export TEST_FAIL=true ;; esac fi diff --git a/testinfra/test.py b/testinfra/test.py index 0b765e407a..2f59707766 100755 --- a/testinfra/test.py +++ b/testinfra/test.py @@ -24,6 +24,7 @@ def get_target_roles(target_host): 'testinfra/app-code', 'testinfra/common', 'testinfra/development/test_xvfb.py'], + "staging": [], "mon-staging": ['testinfra/mon', 'testinfra/common'], "mon-prod": ['testinfra/mon']} @@ -92,6 +93,35 @@ def run_testinfra(target_host, verbose=True): {target_roles} """.lstrip().rstrip() + elif target_host == 'staging': + if "CI_SSH_CONFIG" in os.environ: + ssh_config_path = '--ssh-config ' + os.environ["CI_SSH_CONFIG"] + inventory = "" + junit = """ + --junit-xml=./{target_host}-results.xml \ + --junit-prefix={target_host} \ + """ + else: + ssh_config_path = "" + inventory = """ + --ansible-inventory \ + .vagrant/provisioners/ansible/inventory/vagrant_ansible_inventory \ + """ + junit = "" + testinfra_command_template = """ +testinfra \ + -vv \ + --connection ansible \ + {ssh_config_path} \ + {inventory} \ + {junit} \ + --hosts app-staging,mon-staging \ + {target_roles} +""".format(ssh_config_path=ssh_config_path, + inventory=inventory, + target_roles=" ".join(target_roles), + junit=junit).lstrip().rstrip() + else: ssh_config_path = "" testinfra_command_template = """ From f8a3c2aaada7c9ccdfc735cf13798a82b0c4dcce Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sat, 27 Jan 2018 22:22:47 +0100 Subject: [PATCH 06/19] ossec: notify the journalist about submissions in the past 24h A cron job runs daily on the app server and updates the /var/lib/securedrop/submissions_today.txt file which contains a boolean set to true if submissions were received in the past 24h, ascreated by the manage.py were_there_submissions_today command. The OSSEC agent on the app server runs a command daily, displaying the content of /var/lib/securedrop/submissions_today.txt. The output of the command is sent to the OSSEC server. A new rule is defined on the OSSEC server to send a mail to when the output is received from the OSSEC agent running on the app server. A new procmail rule is definied on the OSSEC server to catch mails encrypt mails containing the /var/lib/securedrop/submissions_today.txt string and send them to the email defined by the journalist_alert_email ansible variable. A new set of (optional) ansible variables, similar to ossec_alert_gpg_public_key, ossec_gpg_fpr, ossec_alert_email are defined: journalist_alert_gpg_public_key, journalist_gpg_fpr, journalist_alert_email. They are used to upload a journalist public key to the OSSEC server and inserted into the send_encrypted_alarm.sh script which handles mails received by procmail. The modified send_encrypted_alarm.sh script takes one argument (journalist or ossec) and dispatches the mail read from stdin to the corresponding recipient. Integration tests are implemented to verify the following: * manage.py were_there_submissions_today * the app OSSEC agent sends a mail to the journalist address * cover all branches of send_encrypted_alarm.sh --- .../roles/app/tasks/setup_cron.yml | 8 + .../roles/ossec/defaults/main.yml | 11 + .../ossec/files/process_submissions_today.sh | 73 ++++++ .../roles/ossec/tasks/configure_server.yml | 23 +- .../ossec/templates/send_encrypted_alarm.sh | 32 ++- .../roles/postfix/files/procmailrc | 11 +- .../var/ossec/etc/ossec.conf | 8 + .../var/ossec/etc/ossec.conf | 8 + .../var/ossec/rules/local_rules.xml | 11 + testinfra/ossec/test_journalist_mail.py | 212 ++++++++++++++++++ testinfra/test.py | 6 +- testinfra/vars/staging.yml | 0 12 files changed, 391 insertions(+), 12 deletions(-) create mode 100755 install_files/ansible-base/roles/ossec/files/process_submissions_today.sh create mode 100644 testinfra/ossec/test_journalist_mail.py create mode 100644 testinfra/vars/staging.yml diff --git a/install_files/ansible-base/roles/app/tasks/setup_cron.yml b/install_files/ansible-base/roles/app/tasks/setup_cron.yml index 49422532ca..ced6fea7ce 100644 --- a/install_files/ansible-base/roles/app/tasks/setup_cron.yml +++ b/install_files/ansible-base/roles/app/tasks/setup_cron.yml @@ -15,3 +15,11 @@ special_time: daily tags: - cron + +- name: Add cron job to update the number of submissions in the past 24h + cron: + name: Update the number of submissions in the past 24h + job: "{{ securedrop_code }}/manage.py were-there-submissions-today" + special_time: daily + tags: + - cron diff --git a/install_files/ansible-base/roles/ossec/defaults/main.yml b/install_files/ansible-base/roles/ossec/defaults/main.yml index 4395f9a537..c0704bfaa4 100644 --- a/install_files/ansible-base/roles/ossec/defaults/main.yml +++ b/install_files/ansible-base/roles/ossec/defaults/main.yml @@ -10,3 +10,14 @@ ossec_group: ossec # will execute tasks conditionally based on the var values. ossec_is_server: False ossec_is_client: False + +# +# The journalist_* variables are optional and may not be present at +# all in the site-specific file. Set an empty string as the default +# for all of them to make it easier for tasks to test if they are set +# or not by comparing to the empty string instead of testing if they +# are defined and if they are set to the empty string. +# +journalist_alert_gpg_public_key: "" +journalist_gpg_fpr: "" +journalist_alert_email: "" diff --git a/install_files/ansible-base/roles/ossec/files/process_submissions_today.sh b/install_files/ansible-base/roles/ossec/files/process_submissions_today.sh new file mode 100755 index 0000000000..899d83df86 --- /dev/null +++ b/install_files/ansible-base/roles/ossec/files/process_submissions_today.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +function send_encrypted_alarm() { + /var/ossec/send_encrypted_alarm.sh "$1" +} + +function main() { + local sender + local stdin + sender=${1:-send_encrypted_alarm} + stdin="$(< /dev/stdin)" + + local count + count=$(echo "$stdin" | perl -ne 'print scalar(<>) and exit if(/ossec: output/);') + if [[ "$count" =~ ^[0-9]+$ ]] ; then + export SUBJECT="Submissions in the past 24h" + # + # whitespaces below are so the size of both messages are exactly the same + # + if [[ "$count" -gt "0" ]] ; then + echo "There has been submission activity in the past 24 hours. " + echo "You should login and check SecureDrop. " + else + echo "There has been no submission activity in the past 24 hours." + echo "You do not need to login to SecureDrop." + fi | $sender journalist + else + export SUBJECT="SecureDrop Submissions Error" + ( + echo "$0 failed to find 0/1 submissions boolean in the following OSSEC alert" + echo + echo "$stdin" + ) | $sender ossec + fi +} + +function test_send_encrypted_alarm() { + echo "$1" + cat +} + +function test_main() { + shopt -s -o xtrace + PS4='${BASH_SOURCE[0]}:$LINENO: ${FUNCNAME[0]}: ' + + echo BUGOUS | main test_send_encrypted_alarm | \ + tee /dev/stderr | \ + grep -q 'failed to find 0/1 submissions boolean' || exit 1 + + ( + echo 'ossec: output' + echo 'NOTANUMBER' + ) | main test_send_encrypted_alarm | tee /dev/stderr | grep -q 'failed to find 0/1 submissions boolean' || exit 1 + + ( + echo 'ossec: output' + echo '1' + ) | main test_send_encrypted_alarm | tee /tmp/submission-yes.txt | grep -q 'There has been submission activity' || exit 1 + + ( + echo 'ossec: output' + echo '0' + ) | main test_send_encrypted_alarm | tee /tmp/submission-no.txt | grep -q 'There has been no submission activity' || exit 1 + + if test "$(stat --format=%s /tmp/submission-no.txt)" != "$(stat --format=%s /tmp/submission-yes.txt)" ; then + echo both files are expected to have exactly the same size, padding must be missing + ls -l /tmp/submission-{yes,no}.txt + tail -n 200 /tmp/submission-{yes,no}.txt + exit 1 + fi +} + +${1:-main} diff --git a/install_files/ansible-base/roles/ossec/tasks/configure_server.yml b/install_files/ansible-base/roles/ossec/tasks/configure_server.yml index e2df707d27..c37790542f 100644 --- a/install_files/ansible-base/roles/ossec/tasks/configure_server.yml +++ b/install_files/ansible-base/roles/ossec/tasks/configure_server.yml @@ -11,8 +11,12 @@ - name: Copy the OSSEC GPG public key for sending encrypted alerts. copy: - src: "{{ ossec_alert_gpg_public_key }}" + src: "{{ item }}" dest: /var/ossec + when: item != '' + with_items: + - "{{ ossec_alert_gpg_public_key }}" + - "{{ journalist_alert_gpg_public_key }}" tags: - gpg @@ -21,11 +25,15 @@ command: > gpg --homedir /var/ossec/.gnupg - --import /var/ossec/{{ ossec_alert_gpg_public_key }} + --import /var/ossec/{{ item }} become: yes become_user: "{{ ossec_group }}" register: add_ossec_gpg_key_result changed_when: "'imported: 1' in add_ossec_gpg_key_result.stderr" + when: item != '' + with_items: + - "{{ ossec_alert_gpg_public_key }}" + - "{{ journalist_alert_gpg_public_key }}" tags: - gpg @@ -40,6 +48,17 @@ - procmail - permissions +- name: Copy script for formatting journalist submission + copy: + src: process_submissions_today.sh + dest: /var/ossec/process_submissions_today.sh + mode: "0550" + owner: root + group: ossec + tags: + - procmail + - permissions + - name: Create OSSEC manager SSL key. command: openssl genrsa -out /var/ossec/etc/sslmanager.key 4096 args: diff --git a/install_files/ansible-base/roles/ossec/templates/send_encrypted_alarm.sh b/install_files/ansible-base/roles/ossec/templates/send_encrypted_alarm.sh index 52d6b8a9e4..f45cec9dec 100644 --- a/install_files/ansible-base/roles/ossec/templates/send_encrypted_alarm.sh +++ b/install_files/ansible-base/roles/ossec/templates/send_encrypted_alarm.sh @@ -16,8 +16,29 @@ set -o pipefail # encryption fail. ossec_alert_text="$(< /dev/stdin)" +# default to environment even if null +env | grep -q JOURNALIST_EMAIL || JOURNALIST_EMAIL='{{ journalist_alert_email }}' +env | grep -q OSSEC_EMAIL || OSSEC_EMAIL='{{ ossec_alert_email }}' + # Primary "send email to Admin" functionality. function send_encrypted_alert() { + local recipient="$1" + local gpg_fpr + local alert_email + local compression + if [[ "$recipient" = "journalist" ]] ; then + gpg_fpr='{{ journalist_gpg_fpr }}' + alert_email="$JOURNALIST_EMAIL" + if [[ "$alert_email" = "" ]] ; then + echo "journalist alert email unset, no notification sent" + return + fi + compression="--compress-algo none" + else + gpg_fpr='{{ ossec_gpg_fpr }}' + alert_email="$OSSEC_EMAIL" + compression="" + fi local encrypted_alert_text # Try to encrypt the alert message. We'll inspect the exit status of the @@ -25,14 +46,14 @@ function send_encrypted_alert() { # failure message. encrypted_alert_text="$(printf "%s" "${ossec_alert_text}" | \ /usr/bin/formail -I '' | \ - /usr/bin/gpg --homedir /var/ossec/.gnupg --trust-model always -ear '{{ ossec_gpg_fpr }}')" + /usr/bin/gpg $compression --homedir /var/ossec/.gnupg --trust-model always -ear "$gpg_fpr")" # Error handling. if [[ -z "${encrypted_alert_text}" || $? -ne 0 ]]; then - send_plaintext_fail_message + send_plaintext_fail_message "$OSSEC_EMAIL" else echo "${encrypted_alert_text}" | \ - /usr/bin/mail -s "$(echo "${SUBJECT}" | sed -r 's/([0-9]{1,3}\.){3}[0-9]{1,3}\s?//g' )" '{{ ossec_alert_email }}' + /usr/bin/mail -s "$(echo "${SUBJECT}" | sed -r 's/([0-9]{1,3}\.){3}[0-9]{1,3}\s?//g' )" "$alert_email" fi } @@ -40,10 +61,11 @@ function send_encrypted_alert() { # Usually a failure is related to GPG balking on the encryption step; # that may be due to a missing pubkey or something reason. function send_plaintext_fail_message() { + local alert_email="$1" printf "Failed to encrypt OSSEC alert. Investigate the mailing configuration on the Monitor Server." | \ /usr/bin/formail -I "" | \ - /usr/bin/mail -s "$(echo "${SUBJECT}" | sed -r 's/([0-9]{1,3}\.){3}[0-9]{1,3}\s?//g' )" '{{ ossec_alert_email }}' + /usr/bin/mail -s "$(echo "${SUBJECT}" | sed -r 's/([0-9]{1,3}\.){3}[0-9]{1,3}\s?//g' )" "$alert_email" } # Encrypt the OSSEC notification and pass to mailer for sending. -send_encrypted_alert +send_encrypted_alert "$@" diff --git a/install_files/ansible-base/roles/postfix/files/procmailrc b/install_files/ansible-base/roles/postfix/files/procmailrc index 729b89d5ba..857d4379b2 100644 --- a/install_files/ansible-base/roles/postfix/files/procmailrc +++ b/install_files/ansible-base/roles/postfix/files/procmailrc @@ -3,6 +3,11 @@ MAILDIR=/var/mail/ DEFAULT=$MAILDIR LOGFILE=/var/log/procmail.log SUBJECT=`formail -xSubject:` -:0 c -*^To:.*root.* -|/var/ossec/send_encrypted_alarm.sh + +:0 B +* /var/lib/securedrop/submissions_today.txt +|/var/ossec/process_submissions_today.sh + +:0 E +* ^To:.*root.* +|/var/ossec/send_encrypted_alarm.sh ossec diff --git a/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf b/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf index 2b72914f24..a5061e48f8 100644 --- a/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf +++ b/install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf @@ -36,6 +36,8 @@ /var/lib/securedrop/db.sqlite + /var/lib/securedrop/submissions_today.txt + /var/securedrop/store /var/ossec/queue @@ -103,6 +105,12 @@ last -n 5 + + full_command + head -1 /var/lib/securedrop/submissions_today.txt | grep '^[0-9]*$' + 86400 + + syslog /var/log/kern.log diff --git a/install_files/securedrop-ossec-server/var/ossec/etc/ossec.conf b/install_files/securedrop-ossec-server/var/ossec/etc/ossec.conf index 771fb23b7e..3430cf0d29 100644 --- a/install_files/securedrop-ossec-server/var/ossec/etc/ossec.conf +++ b/install_files/securedrop-ossec-server/var/ossec/etc/ossec.conf @@ -130,6 +130,14 @@ + + + root@localhost + 400600 + + + + rules_config.xml diff --git a/install_files/securedrop-ossec-server/var/ossec/rules/local_rules.xml b/install_files/securedrop-ossec-server/var/ossec/rules/local_rules.xml index e182d5509f..97f0cfe62e 100644 --- a/install_files/securedrop-ossec-server/var/ossec/rules/local_rules.xml +++ b/install_files/securedrop-ossec-server/var/ossec/rules/local_rules.xml @@ -190,3 +190,14 @@ no_email_alert + + + + 530 + alert_by_email + ossec: output: 'head -1 /var/lib/securedrop/submissions_today.txt + Boolean value indicating if there were submissions in the past 24h. + + + + diff --git a/testinfra/ossec/test_journalist_mail.py b/testinfra/ossec/test_journalist_mail.py new file mode 100644 index 0000000000..69dd319906 --- /dev/null +++ b/testinfra/ossec/test_journalist_mail.py @@ -0,0 +1,212 @@ +import pytest +import testinfra +import time + + +class TestBase(object): + + @pytest.fixture(autouse=True) + def only_mon_staging_sudo(self, host): + if host.backend.host != 'mon-staging': + pytest.skip() + + with host.sudo(): + yield + + def ansible(self, host, module, parameters): + r = host.ansible(module, parameters, check=False) + assert 'exception' not in r + + def run(self, host, cmd): + print(host.backend.host + " running: " + cmd) + r = host.run(cmd) + print(r.stdout) + print(r.stderr) + return r.rc == 0 + + def wait_for(self, fun): + success = False + for d in (1, 2, 4, 8, 16, 32, 64): + if fun(): + success = True + break + time.sleep(d) + return success + + def wait_for_command(self, host, cmd): + return self.wait_for(lambda: self.run(host, cmd)) + + # + # implementation note: we do not use host.ansible("service", ... + # because it only works for services in /etc/init and not those + # legacy only found in /etc/init.d such as postfix + # + def service_started(self, host, name): + assert self.run(host, "service {name} start".format(name=name)) + assert self.wait_for_command( + host, + "service {name} status | grep -q 'is running'".format(name=name)) + + def service_restarted(self, host, name): + assert self.run(host, "service {name} restart".format(name=name)) + assert self.wait_for_command( + host, + "service {name} status | grep -q 'is running'".format(name=name)) + + def service_stopped(self, host, name): + assert self.run(host, "service {name} stop".format(name=name)) + assert self.wait_for_command( + host, + "service {name} status | grep -q 'not running'".format(name=name)) + + +class TestJournalistMail(TestBase): + + def test_procmail(self, host): + self.service_started(host, "postfix") + today_payload = ( + 'ossec: output: head -1 /var/lib/securedrop/submissions_today.txt' + '\n1234') + for (destination, payload) in ( + ('journalist', today_payload), + ('ossec', 'MYGREATPAYLOAD')): + assert self.run(host, "postsuper -d ALL") + assert self.run( + host, + "echo -e '{payload}' | " + "mail -s 'abc' root@localhost".format(payload=payload)) + assert self.wait_for_command( + host, + "mailq | grep -q {destination}@ossec.test".format( + destination=destination)) + self.service_stopped(host, "postfix") + + def test_process_submissions_today(self, host): + self.run(host, "/var/ossec/process_submissions_today.sh test_main") + + def test_send_encrypted_alert(self, host): + self.service_started(host, "postfix") + src = "install_files/ansible-base/roles/ossec/files/test_admin_key.sec" + self.ansible(host, "copy", + "dest=/tmp/test_admin_key.sec src={src}".format(src=src)) + + self.run(host, "gpg --homedir /var/ossec/.gnupg" + " --import /tmp/test_admin_key.sec") + + def trigger(who, payload): + assert self.run( + host, "! mailq | grep -q {who}@ossec.test".format(who=who)) + assert self.run( + host, + """ + ( echo 'Subject: TEST' ; echo ; echo -e '{payload}' ) | \ + /var/ossec/send_encrypted_alarm.sh {who} + """.format(who=who, payload=payload)) + assert self.wait_for_command( + host, "mailq | grep -q {who}@ossec.test".format(who=who)) + + # + # encrypted mail to journalist or ossec contact + # + for (who, payload, expected) in ( + ('journalist', 'ossec: output\n1', '1'), + ('ossec', 'MYGREATPAYLOAD', 'MYGREATPAYLOAD')): + assert self.run(host, "postsuper -d ALL") + trigger(who, payload) + assert self.run( + host, + """ + job=$(mailq | sed -n -e '2p' | cut -f1 -d ' ') + postcat -q $job | tee /dev/stderr | \ + gpg --homedir /var/ossec/.gnupg --decrypt 2>&1 | \ + grep -q {expected} + """.format(expected=expected)) + # + # failure to encrypt must trigger an emergency mail to ossec contact + # + try: + assert self.run(host, "postsuper -d ALL") + assert self.run(host, "mv /usr/bin/gpg /usr/bin/gpg.save") + trigger(who, 'MYGREATPAYLOAD') + assert self.run( + host, + """ + job=$(mailq | sed -n -e '2p' | cut -f1 -d ' ') + postcat -q $job | grep -q 'Failed to encrypt OSSEC alert' + """) + finally: + assert self.run(host, "mv /usr/bin/gpg.save /usr/bin/gpg") + self.service_stopped(host, "postfix") + + def test_missing_journalist_alert(self, host): + # + # missing journalist mail does nothing + # + assert self.run( + host, + """ + JOURNALIST_EMAIL= \ + bash -x /var/ossec/send_encrypted_alarm.sh journalist | \ + tee /dev/stderr | \ + grep -q 'no notification sent' + """) + + # https://ossec-docs.readthedocs.io/en/latest/manual/rules-decoders/testing.html + def test_ossec_rule_journalist(self, host): + assert self.run(host, """ + set -ex + l="ossec: output: 'head -1 /var/lib/securedrop/submissions_today.txt" + echo "$l" | /var/ossec/bin/ossec-logtest + echo "$l" | /var/ossec/bin/ossec-logtest -U '400600:1:ossec' + """) + + def test_journalist_mail_notification(self, host): + mon = host + app = testinfra.host.Host.get_host( + 'ansible://app-staging', + ansible_inventory=host.backend.ansible_inventory) + # + # run ossec & postfix on mon + # + self.service_started(mon, "postfix") + self.service_started(mon, "ossec") + + # + # ensure the submission_today.txt file exists + # + with app.sudo(): + assert self.run(app, """ + cd /var/www/securedrop + ./manage.py were-there-submissions-today + test -f /var/lib/securedrop/submissions_today.txt + """) + + # + # empty the mailq on mon in case there were leftovers + # + assert self.run(mon, "postsuper -d ALL") + + # + # the command fires every time ossec starts, + # regardless of the frequency + # + with app.sudo(): + self.service_restarted(app, "ossec") + + # + # wait until at exactly one notification is sent + # + assert self.wait_for_command( + mon, + "mailq | grep -q journalist@ossec.test") + assert self.run( + mon, + "test 1 = $(mailq | grep journalist@ossec.test | wc -l)") + + # + # teardown the ossec and postfix on mon and app + # + self.service_stopped(mon, "postfix") + self.service_stopped(mon, "ossec") + with app.sudo(): + self.service_stopped(app, "ossec") diff --git a/testinfra/test.py b/testinfra/test.py index 2f59707766..f5c326ca85 100755 --- a/testinfra/test.py +++ b/testinfra/test.py @@ -24,7 +24,7 @@ def get_target_roles(target_host): 'testinfra/app-code', 'testinfra/common', 'testinfra/development/test_xvfb.py'], - "staging": [], + "staging": ['testinfra/ossec'], "mon-staging": ['testinfra/mon', 'testinfra/common'], "mon-prod": ['testinfra/mon']} @@ -112,6 +112,7 @@ def run_testinfra(target_host, verbose=True): testinfra \ -vv \ --connection ansible \ + {testinfra_args} \ {ssh_config_path} \ {inventory} \ {junit} \ @@ -120,7 +121,8 @@ def run_testinfra(target_host, verbose=True): """.format(ssh_config_path=ssh_config_path, inventory=inventory, target_roles=" ".join(target_roles), - junit=junit).lstrip().rstrip() + junit=junit, + testinfra_args=os.environ.get('TESTINFRA_ARGS', '')) else: ssh_config_path = "" diff --git a/testinfra/vars/staging.yml b/testinfra/vars/staging.yml new file mode 100644 index 0000000000..e69de29bb2 From 347cdd9b2d121d3039d8a09dc7e970925e781d17 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sat, 27 Jan 2018 23:39:39 +0100 Subject: [PATCH 07/19] testinfra: remove some OSSEC tests duplicating Ansible --- testinfra/mon/test_ossec.py | 87 ------------------------------------- 1 file changed, 87 deletions(-) diff --git a/testinfra/mon/test_ossec.py b/testinfra/mon/test_ossec.py index b5a3c78bab..2a76b5d5b2 100644 --- a/testinfra/mon/test_ossec.py +++ b/testinfra/mon/test_ossec.py @@ -5,21 +5,6 @@ securedrop_test_vars = pytest.securedrop_test_vars -@pytest.mark.parametrize('package', [ - 'mailutils', - 'ossec-server', - 'postfix', - 'procmail', - 'securedrop-ossec-server', -]) -def test_ossec_package(Package, package): - """ - Ensure required packages for OSSEC are installed. - Includes mail utilities and the FPF-maintained metapackage. - """ - assert Package(package).is_installed - - def test_ossec_connectivity(Command, Sudo): """ Ensure ossec-server machine has active connection to the ossec-agent. @@ -34,44 +19,6 @@ def test_ossec_connectivity(Command, Sudo): assert c == desired_output -def test_ossec_gnupg_homedir(File, Sudo): - """ ensure ossec gpg homedir exists """ - with Sudo(): - f = File("/var/ossec/.gnupg") - assert f.is_directory - assert f.user == "ossec" - assert oct(f.mode) == "0700" - - -# Permissions don't match between Ansible and OSSEC deb packages postinst. -@pytest.mark.xfail -def test_ossec_gnupg(File, Sudo): - """ - Ensures the test Admin GPG public key is present as file. - Does not check that it's added to the keyring for the ossec user; - that's handled by a separate test. - """ - with Sudo(): - f = File("/var/ossec/test_admin_key.pub") - assert f.is_file - assert oct(f.mode) == "0644" - - -def test_ossec_pubkey_in_keyring(Command, Sudo): - """ - Ensure the test Admin GPG public key exists in the keyring - within the ossec home directory. - """ - ossec_gpg_pubkey_info = """pub 2048R/B5E53711 2018-01-25 -uid SecureDrop admin key for tests (do not use in production) -sub 2048R/EC1DF5D0 2018-01-25""" # noqa - with Sudo("ossec"): - c = Command.check_output( - "gpg --homedir /var/ossec/.gnupg " - "--list-keys 53E1113AC1F25027BA5D475B1141E2BBB5E53711") - assert c == ossec_gpg_pubkey_info - - # Permissions don't match between Ansible and OSSEC deb packages postinst. @pytest.mark.xfail @pytest.mark.parametrize('keyfile', [ @@ -96,40 +43,6 @@ def test_ossec_keyfiles(File, Sudo, keyfile): assert f.group == "ossec" -@pytest.mark.parametrize('setting', [ - 'VERBOSE=yes', - 'MAILDIR=/var/mail/', - 'DEFAULT=$MAILDIR', - 'LOGFILE=/var/log/procmail.log', - 'SUBJECT=`formail -xSubject:`', - ':0 c', - '*^To:.*root.*', - '|/var/ossec/send_encrypted_alarm.sh', -]) -def test_procmail_settings(File, Sudo, setting): - """ - Ensure procmail settings are correct. These config lines determine - how the OSSEC email alerts are encrypted and then passed off for sending. - """ - # Sudo is required to traverse the /var/ossec directory. - with Sudo(): - f = File("/var/ossec/.procmailrc") - assert f.contains('^{}$'.format(setting)) - - -# Permissions don't match between Ansible and OSSEC deb packages postinst. -@pytest.mark.xfail -def test_procmail_attrs(File, Sudo): - """ - Ensure procmail file attributes are specified correctly. - """ - with Sudo(): - f = File("/var/ossec/.procmailrc") - assert f.is_file - assert f.user == "ossec" - assert oct(f.mode) == "0440" - - # Permissions don't match between Ansible and OSSEC deb packages postinst. @pytest.mark.xfail def test_procmail_log(File, Sudo): From c579d89bdafeca46bf2424b215990a0278feff42 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sun, 28 Jan 2018 00:31:14 +0100 Subject: [PATCH 08/19] testinfra: remove some postfix tests duplicating Ansible The original is at install_files/ansible-base/roles/postfix/templates/main.cf --- testinfra/mon/test_postfix.py | 42 ----------------------------------- 1 file changed, 42 deletions(-) diff --git a/testinfra/mon/test_postfix.py b/testinfra/mon/test_postfix.py index 65ce4841cd..cb36f84006 100644 --- a/testinfra/mon/test_postfix.py +++ b/testinfra/mon/test_postfix.py @@ -25,48 +25,6 @@ def test_postfix_headers(File, header): assert re.search(regex, f.content, re.M) -@pytest.mark.parametrize('setting', [ - 'relayhost = [smtp.gmail.com]:587', - 'smtp_sasl_auth_enable = yes', - 'smtp_sasl_password_maps = hash:/etc/postfix/sasl_passwd', - 'smtp_sasl_security_options = noanonymous', - 'smtp_use_tls = yes', - 'smtp_tls_session_cache_database = btree:${data_directory}/smtp_scache', - 'smtp_tls_security_level = secure', - 'smtp_tls_CApath = /etc/ssl/certs', - 'smtp_tls_ciphers = high', - 'smtp_tls_protocols = TLSv1.2 TLSv1.1 TLSv1 !SSLv3 !SSLv2', - 'myhostname = ossec.server', - 'myorigin = $myhostname', - 'smtpd_banner = $myhostname ESMTP $mail_name (Ubuntu)', - 'biff = no', - 'append_dot_mydomain = no', - 'readme_directory = no', - 'smtp_header_checks = regexp:/etc/postfix/header_checks', - 'mailbox_command = /usr/bin/procmail', - 'inet_interfaces = loopback-only', - 'alias_maps = hash:/etc/aliases', - 'alias_database = hash:/etc/aliases', - 'mydestination = $myhostname, localhost.localdomain , localhost', - 'mynetworks = 127.0.0.0/8 [::ffff:127.0.0.0]/104 [::1]/128', - 'mailbox_size_limit = 0', - 'recipient_delimiter = +', -]) -def test_postfix_settings(File, setting): - """ - Check all postfix configuration lines. There are technically multiple - configuration paths regarding the TLS settings, particularly the - fingerprint verification logic, but only the base default config is tested - currently. - """ - f = File("/etc/postfix/main.cf") - assert f.is_file - assert f.user == 'root' - assert oct(f.mode) == "0644" - regex = '^{}$'.format(re.escape(setting)) - assert re.search(regex, f.content, re.M) - - def test_postfix_generic_maps(File): """ Regression test to check that generic Postfix maps are not configured From 78fe72a93ba8b3834b6e5d547f4e30b216cc0cb5 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Sun, 28 Jan 2018 23:26:00 +0100 Subject: [PATCH 09/19] docs: admin and journalist guide regarding daily notifications --- docs/install.rst | 15 +++++++++++++++ docs/journalist.rst | 19 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/docs/install.rst b/docs/install.rst index fd689fcfea..a9cd8b8866 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -60,6 +60,21 @@ continuing: can add more later) - The username of the system admin +You can also, optionally, configure a :doc:`daily notifications +` about whether or not submission activity occurred in the +past 24 hours. They are sent via email so journalists know if it is +worth checking the *Journalist Interface*. For this you will need: + +- The journalist alerts GPG key +- The journalist alerts GPG key fingerprint +- The email address that will receive the journalist alerts + +.. note:: It is not possible to specify multiple email addresses. If + there are more than one recipient, an alias or a mailing + list must be created. All journalist subscribed must share + the GPG private key, it is not possible to specify multiple + GPG keys. + You will have to copy the following required files to ``install_files/ansible-base``: diff --git a/docs/journalist.rst b/docs/journalist.rst index 02d5563e85..03231fab3c 100644 --- a/docs/journalist.rst +++ b/docs/journalist.rst @@ -57,6 +57,25 @@ Journalist Interface `.) |Journalist Interface Login| +Daily journalist alerts about submissions +----------------------------------------- + +When a SecureDrop has little activity and receives only a few +submissions every other week, checking the *Journalist Interface* +daily only to find there is nothing is a burden. It is more convenient +for journalists to be notified daily via encrypted email about whether +or not there has been submission activity in the past 24 hours. + +If the email shows submissions were received, the journalist can +connect to the *Journalist Interface* to get them. + +This is an optional feature that must be activated :doc:`by the +administrator `. In the simplest case a journalist provides +her/his email and GPG public key to the admin. If a team of journalist +wants to receive these daily alerts, they should share a GPG key and +ask the admin to setup a mail alias (SecureDrop does not provide that +service) so they all receive the alerts and are able to decrypt them. + Interacting With Sources ------------------------ From cf851178acdb392b50da5199d4b1d716b5bff371 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Wed, 31 Jan 2018 18:09:10 +0100 Subject: [PATCH 10/19] securedrop-admin: split ValidateEmail out of ValidateOSSECEmail --- admin/securedrop_admin/__init__.py | 20 ++++++++++++++++---- admin/tests/test_securedrop-admin.py | 9 +++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py index d6c4c6b0a5..55645f244c 100755 --- a/admin/securedrop_admin/__init__.py +++ b/admin/securedrop_admin/__init__.py @@ -192,14 +192,26 @@ def validate(self, document): raise ValidationError( message="Password for OSSEC email account must be strong") - class ValidateOSSECEmail(Validator): + class ValidateEmail(Validator): def validate(self, document): text = document.text - if text and '@' in text and 'ossec@ossec.test' != text: + if text == '': + raise ValidationError( + message=("Must not be empty")) + if '@' not in text: + raise ValidationError( + message=("Must contain a @")) + return True + + class ValidateOSSECEmail(ValidateEmail): + def validate(self, document): + super(SiteConfig.ValidateOSSECEmail, self).validate(document) + text = document.text + if 'ossec@ossec.test' != text: return True raise ValidationError( - message=("Must contain a @ and be set to " - "something other than ossec@ossec.test")) + message=("Must be set to something other than " + "ossec@ossec.test")) def __init__(self, args): self.args = args diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py index 4bdca48862..1a9e5fbfc0 100644 --- a/admin/tests/test_securedrop-admin.py +++ b/admin/tests/test_securedrop-admin.py @@ -178,14 +178,19 @@ def test_validate_ossec_password(self): with pytest.raises(ValidationError): validator.validate(Document('short')) - def test_validate_ossec_email(self): - validator = securedrop_admin.SiteConfig.ValidateOSSECEmail() + def test_validate_email(self): + validator = securedrop_admin.SiteConfig.ValidateEmail() assert validator.validate(Document('good@mail.com')) with pytest.raises(ValidationError): validator.validate(Document('badmail')) with pytest.raises(ValidationError): validator.validate(Document('')) + + def test_validate_ossec_email(self): + validator = securedrop_admin.SiteConfig.ValidateOSSECEmail() + + assert validator.validate(Document('good@mail.com')) with pytest.raises(ValidationError) as e: validator.validate(Document('ossec@ossec.test')) assert 'something other than ossec@ossec.test' in e.value.message From ec4a48a5c584f9597cbea52e07cf3ee44c138040 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Wed, 31 Jan 2018 18:14:01 +0100 Subject: [PATCH 11/19] securedrop-admin: test_validate_gpg_key must verify all fingerprints --- admin/tests/test_securedrop-admin.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py index 1a9e5fbfc0..02cd3cdb86 100644 --- a/admin/tests/test_securedrop-admin.py +++ b/admin/tests/test_securedrop-admin.py @@ -359,8 +359,7 @@ def test_validate_gpg_key(self, caplog): args = argparse.Namespace(site_config='INVALID', ansible_path='tests/files', app_path=dirname(__file__)) - site_config = securedrop_admin.SiteConfig(args) - site_config.config = { + good_config = { 'securedrop_app_gpg_public_key': 'test_journalist_key.pub', @@ -373,11 +372,18 @@ def test_validate_gpg_key(self, caplog): 'ossec_gpg_fpr': '65A1B5FF195B56353CC63DFFCC40EF1228271441', } + site_config = securedrop_admin.SiteConfig(args) + site_config.config = good_config assert site_config.validate_gpg_keys() - site_config.config['ossec_gpg_fpr'] = 'FAIL' - with pytest.raises(securedrop_admin.FingerprintException) as e: - site_config.validate_gpg_keys() - assert 'FAIL does not match' in e.value.message + + for key in ('securedrop_app_gpg_fingerprint', + 'ossec_gpg_fpr'): + bad_config = good_config.copy() + bad_config[key] = 'FAIL' + site_config.config = bad_config + with pytest.raises(securedrop_admin.FingerprintException) as e: + site_config.validate_gpg_keys() + assert 'FAIL does not match' in e.value.message @mock.patch('securedrop_admin.SiteConfig.validated_input', side_effect=lambda p, d, v, t: d) From 85ec7c439a932f29b01338d990e03349d5fe2ef4 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Wed, 31 Jan 2018 18:15:52 +0100 Subject: [PATCH 12/19] securedrop-admin: implement verify_desc_consistency_optional To verify the consistency of a sdconfig value in the case where it is not mandatory. --- admin/tests/test_securedrop-admin.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py index 02cd3cdb86..8a4170b961 100644 --- a/admin/tests/test_securedrop-admin.py +++ b/admin/tests/test_securedrop-admin.py @@ -448,11 +448,15 @@ def get_desc(self, site_config, var): if desc[0] == var: return desc - def verify_desc_consistency(self, site_config, desc): + def verify_desc_consistency_optional(self, site_config, desc): (var, default, etype, prompt, validator, transform) = desc # verify the default passes validation assert site_config.user_prompt_config_one(desc, None) == default assert type(default) == etype + + def verify_desc_consistency(self, site_config, desc): + self.verify_desc_consistency_optional(site_config, desc) + (var, default, etype, prompt, validator, transform) = desc with pytest.raises(ValidationError): site_config.user_prompt_config_one(desc, '') From a595209d75017b8df92de9c5d79331ea87ca3dfb Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Wed, 31 Jan 2018 18:17:47 +0100 Subject: [PATCH 13/19] securedrop-admin: cosmetic cleanup moving a constant out of a loop --- admin/securedrop_admin/__init__.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py index 55645f244c..d5734dcf80 100755 --- a/admin/securedrop_admin/__init__.py +++ b/admin/securedrop_admin/__init__.py @@ -353,10 +353,10 @@ def validate_gpg_keys(self): ('ossec_alert_gpg_public_key', 'ossec_gpg_fpr')) + validate = os.path.join( + os.path.dirname(__file__), '..', 'bin', + 'validate-gpg-key.sh') for (public_key, fingerprint) in keys: - validate = os.path.join( - os.path.dirname(__file__), '..', 'bin', - 'validate-gpg-key.sh') public_key = os.path.join(self.args.ansible_path, self.config[public_key]) fingerprint = self.config[fingerprint] From 8a29dec51fa8613861f6b9f61b1d9be5afc95a86 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Wed, 31 Jan 2018 18:20:34 +0100 Subject: [PATCH 14/19] securedrop-admin: add sdconfig journalist_alert_gpg_public_key input --- admin/securedrop_admin/__init__.py | 11 +++++++++++ admin/tests/test_securedrop-admin.py | 9 +++++++++ 2 files changed, 20 insertions(+) diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py index d5734dcf80..9722ffdd5c 100755 --- a/admin/securedrop_admin/__init__.py +++ b/admin/securedrop_admin/__init__.py @@ -122,6 +122,13 @@ def validate(self, document): raise ValidationError( message=path + ' file does not exist') + class ValidateOptionalPath(ValidatePath): + def validate(self, document): + if document.text == '': + return True + return super(SiteConfig.ValidateOptionalPath, self).validate( + document) + class ValidateYesNo(Validator): def validate(self, document): text = document.text.lower() @@ -269,6 +276,10 @@ def __init__(self, args): u'Admin email address for receiving OSSEC alerts', SiteConfig.ValidateOSSECEmail(), None], + ['journalist_alert_gpg_public_key', '', str, + u'Local filepath to journalist alerts GPG public key (optional)', + SiteConfig.ValidateOptionalPath(self.args.ansible_path), + None], ['smtp_relay', "smtp.gmail.com", str, u'SMTP relay for sending OSSEC alerts', SiteConfig.ValidateNotEmpty(), diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py index 8a4170b961..58f645d26f 100644 --- a/admin/tests/test_securedrop-admin.py +++ b/admin/tests/test_securedrop-admin.py @@ -278,6 +278,13 @@ def test_validate_path(self): with pytest.raises(ValidationError): validator.validate(Document("")) + def test_validate_optional_path(self): + mydir = dirname(__file__) + myfile = basename(__file__) + validator = securedrop_admin.SiteConfig.ValidateOptionalPath(mydir) + assert validator.validate(Document(myfile)) + assert validator.validate(Document("")) + def test_validate_yes_no(self): validator = securedrop_admin.SiteConfig.ValidateYesNo() with pytest.raises(ValidationError): @@ -498,6 +505,8 @@ def verify_desc_consistency_allow_empty(self, site_config, desc): verify_prompt_ossec_alert_gpg_public_key = verify_desc_consistency verify_prompt_ossec_gpg_fpr = verify_prompt_fingerprint verify_prompt_ossec_alert_email = verify_prompt_not_empty + verify_prompt_journalist_alert_gpg_public_key = ( + verify_desc_consistency_optional) verify_prompt_smtp_relay = verify_prompt_not_empty verify_prompt_smtp_relay_port = verify_desc_consistency verify_prompt_sasl_domain = verify_desc_consistency_allow_empty From 23e995587dee5437bfd0355150c6c84414704df0 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Wed, 31 Jan 2018 18:22:27 +0100 Subject: [PATCH 15/19] securedrop-admin: add sdconfig journalist_gpg_fpr input --- admin/securedrop_admin/__init__.py | 12 ++++++++++++ admin/tests/test_securedrop-admin.py | 14 ++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py index 9722ffdd5c..ec24c766a5 100755 --- a/admin/securedrop_admin/__init__.py +++ b/admin/securedrop_admin/__init__.py @@ -150,6 +150,13 @@ def validate(self, document): message='fingerprints must be 40 hexadecimal characters') return True + class ValidateOptionalFingerprint(ValidateFingerprint): + def validate(self, document): + if document.text == '': + return True + return super(SiteConfig.ValidateOptionalFingerprint, + self).validate(document) + class ValidateInt(Validator): def validate(self, document): if re.match('\d+$', document.text): @@ -280,6 +287,11 @@ def __init__(self, args): u'Local filepath to journalist alerts GPG public key (optional)', SiteConfig.ValidateOptionalPath(self.args.ansible_path), None], + ['journalist_gpg_fpr', '', str, + u'Full fingerprint for the journalist alerts ' + u'GPG public key (optional)', + SiteConfig.ValidateOptionalFingerprint(), + self.sanitize_fingerprint], ['smtp_relay', "smtp.gmail.com", str, u'SMTP relay for sending OSSEC alerts', SiteConfig.ValidateNotEmpty(), diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py index 58f645d26f..c102852e42 100644 --- a/admin/tests/test_securedrop-admin.py +++ b/admin/tests/test_securedrop-admin.py @@ -321,6 +321,12 @@ def test_validate_fingerprint(self): "zzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzzz")) assert '40 hexadecimal' in e.value.message + def test_validate_optional_fingerprint(self): + validator = securedrop_admin.SiteConfig.ValidateOptionalFingerprint() + assert validator.validate(Document( + "012345678901234567890123456789ABCDEFABCD")) + assert validator.validate(Document("")) + def test_sanitize_fingerprint(self): args = argparse.Namespace(site_config='DOES_NOT_EXIST', ansible_path='.', @@ -489,8 +495,7 @@ def verify_prompt_not_empty(self, site_config, desc): with pytest.raises(ValidationError): site_config.user_prompt_config_one(desc, '') - def verify_prompt_fingerprint(self, site_config, desc): - self.verify_prompt_not_empty(site_config, desc) + def verify_prompt_fingerprint_optional(self, site_config, desc): fpr = "0123456 789012 34567890123456789ABCDEFABCD" clean_fpr = site_config.sanitize_fingerprint(fpr) assert site_config.user_prompt_config_one(desc, fpr) == clean_fpr @@ -501,12 +506,17 @@ def verify_desc_consistency_allow_empty(self, site_config, desc): assert site_config.user_prompt_config_one(desc, None) == default assert type(default) == etype + def verify_prompt_fingerprint(self, site_config, desc): + self.verify_prompt_not_empty(site_config, desc) + self.verify_prompt_fingerprint_optional(site_config, desc) + verify_prompt_securedrop_app_gpg_fingerprint = verify_prompt_fingerprint verify_prompt_ossec_alert_gpg_public_key = verify_desc_consistency verify_prompt_ossec_gpg_fpr = verify_prompt_fingerprint verify_prompt_ossec_alert_email = verify_prompt_not_empty verify_prompt_journalist_alert_gpg_public_key = ( verify_desc_consistency_optional) + verify_prompt_journalist_gpg_fpr = verify_prompt_fingerprint_optional verify_prompt_smtp_relay = verify_prompt_not_empty verify_prompt_smtp_relay_port = verify_desc_consistency verify_prompt_sasl_domain = verify_desc_consistency_allow_empty From 8b841bd90082ae04d529efeaf3f3638053189fbb Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Wed, 31 Jan 2018 18:23:50 +0100 Subject: [PATCH 16/19] securedrop-admin: add sdconfig journalist_alert_email input --- admin/securedrop_admin/__init__.py | 11 +++++++++++ admin/tests/test_securedrop-admin.py | 7 +++++++ 2 files changed, 18 insertions(+) diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py index ec24c766a5..c2f2ee7f66 100755 --- a/admin/securedrop_admin/__init__.py +++ b/admin/securedrop_admin/__init__.py @@ -227,6 +227,13 @@ def validate(self, document): message=("Must be set to something other than " "ossec@ossec.test")) + class ValidateOptionalEmail(ValidateEmail): + def validate(self, document): + if document.text == '': + return True + return super(SiteConfig.ValidateOptionalEmail, self).validate( + document) + def __init__(self, args): self.args = args translations = SiteConfig.Locales( @@ -292,6 +299,10 @@ def __init__(self, args): u'GPG public key (optional)', SiteConfig.ValidateOptionalFingerprint(), self.sanitize_fingerprint], + ['journalist_alert_email', '', str, + u'Email address for receiving journalist alerts (optional)', + SiteConfig.ValidateOptionalEmail(), + None], ['smtp_relay', "smtp.gmail.com", str, u'SMTP relay for sending OSSEC alerts', SiteConfig.ValidateNotEmpty(), diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py index c102852e42..79681a225e 100644 --- a/admin/tests/test_securedrop-admin.py +++ b/admin/tests/test_securedrop-admin.py @@ -195,6 +195,12 @@ def test_validate_ossec_email(self): validator.validate(Document('ossec@ossec.test')) assert 'something other than ossec@ossec.test' in e.value.message + def test_validate_optional_email(self): + validator = securedrop_admin.SiteConfig.ValidateOptionalEmail() + + assert validator.validate(Document('good@mail.com')) + assert validator.validate(Document('')) + def test_is_tails(self): validator = securedrop_admin.SiteConfig.ValidateDNS() with mock.patch('subprocess.check_output', return_value='Tails'): @@ -517,6 +523,7 @@ def verify_prompt_fingerprint(self, site_config, desc): verify_prompt_journalist_alert_gpg_public_key = ( verify_desc_consistency_optional) verify_prompt_journalist_gpg_fpr = verify_prompt_fingerprint_optional + verify_prompt_journalist_alert_email = verify_desc_consistency_optional verify_prompt_smtp_relay = verify_prompt_not_empty verify_prompt_smtp_relay_port = verify_desc_consistency verify_prompt_sasl_domain = verify_desc_consistency_allow_empty From a30499b374c79258796f0bc2b8e890d5f37a2fc5 Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Wed, 31 Jan 2018 18:32:43 +0100 Subject: [PATCH 17/19] securedrop-admin: when present the journalist alert key must be verified --- admin/securedrop_admin/__init__.py | 8 +++++++- admin/tests/test_securedrop-admin.py | 9 ++++++++- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py index c2f2ee7f66..e68f9fbb23 100755 --- a/admin/securedrop_admin/__init__.py +++ b/admin/securedrop_admin/__init__.py @@ -386,11 +386,17 @@ def validate_gpg_keys(self): 'securedrop_app_gpg_fingerprint'), ('ossec_alert_gpg_public_key', - 'ossec_gpg_fpr')) + 'ossec_gpg_fpr'), + + ('journalist_alert_gpg_public_key', + 'journalist_gpg_fpr')) validate = os.path.join( os.path.dirname(__file__), '..', 'bin', 'validate-gpg-key.sh') for (public_key, fingerprint) in keys: + if (self.config[public_key] == '' and + self.config[fingerprint] == ''): + continue public_key = os.path.join(self.args.ansible_path, self.config[public_key]) fingerprint = self.config[fingerprint] diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py index 79681a225e..037731b630 100644 --- a/admin/tests/test_securedrop-admin.py +++ b/admin/tests/test_securedrop-admin.py @@ -390,13 +390,20 @@ def test_validate_gpg_key(self, caplog): 'ossec_gpg_fpr': '65A1B5FF195B56353CC63DFFCC40EF1228271441', + + 'journalist_alert_gpg_public_key': + 'test_journalist_key.pub', + + 'journalist_gpg_fpr': + '65A1B5FF195B56353CC63DFFCC40EF1228271441', } site_config = securedrop_admin.SiteConfig(args) site_config.config = good_config assert site_config.validate_gpg_keys() for key in ('securedrop_app_gpg_fingerprint', - 'ossec_gpg_fpr'): + 'ossec_gpg_fpr', + 'journalist_gpg_fpr'): bad_config = good_config.copy() bad_config[key] = 'FAIL' site_config.config = bad_config From 0713b7692f4b8c69ccae117c69cc4b1fb2e8843b Mon Sep 17 00:00:00 2001 From: Loic Dachary Date: Wed, 31 Jan 2018 18:32:59 +0100 Subject: [PATCH 18/19] securedrop-admin: when a journalist key is present, the email is required --- admin/securedrop_admin/__init__.py | 22 +++++++++++++++++ admin/tests/test_securedrop-admin.py | 35 ++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py index e68f9fbb23..5a88cb9573 100755 --- a/admin/securedrop_admin/__init__.py +++ b/admin/securedrop_admin/__init__.py @@ -43,6 +43,10 @@ class FingerprintException(Exception): pass +class JournalistAlertEmailException(Exception): + pass + + class SiteConfig(object): class ValidateNotEmpty(Validator): @@ -341,6 +345,7 @@ def update_config(self): self.config.update(self.user_prompt_config()) self.save() self.validate_gpg_keys() + self.validate_journalist_alert_email() return True def user_prompt_config(self): @@ -412,6 +417,23 @@ def validate_gpg_keys(self): "the public key {}".format(public_key)) return True + def validate_journalist_alert_email(self): + if (self.config['journalist_alert_gpg_public_key'] == '' and + self.config['journalist_gpg_fpr'] == ''): + return True + + class Document(object): + def __init__(self, text): + self.text = text + + try: + SiteConfig.ValidateEmail().validate(Document( + self.config['journalist_alert_email'])) + except ValidationError as e: + raise JournalistAlertEmailException( + "journalist alerts email: " + e.message) + return True + def exists(self): return os.path.exists(self.args.site_config) diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py index 037731b630..9b713c225e 100644 --- a/admin/tests/test_securedrop-admin.py +++ b/admin/tests/test_securedrop-admin.py @@ -411,6 +411,41 @@ def test_validate_gpg_key(self, caplog): site_config.validate_gpg_keys() assert 'FAIL does not match' in e.value.message + def test_journalist_alert_email(self): + args = argparse.Namespace(site_config='INVALID', + ansible_path='tests/files', + app_path=dirname(__file__)) + site_config = securedrop_admin.SiteConfig(args) + site_config.config = { + 'journalist_alert_gpg_public_key': + '', + + 'journalist_gpg_fpr': + '', + } + assert site_config.validate_journalist_alert_email() + site_config.config = { + 'journalist_alert_gpg_public_key': + 'test_journalist_key.pub', + + 'journalist_gpg_fpr': + '65A1B5FF195B56353CC63DFFCC40EF1228271441', + } + site_config.config['journalist_alert_email'] = '' + with pytest.raises( + securedrop_admin.JournalistAlertEmailException) as e: + site_config.validate_journalist_alert_email() + assert 'not be empty' in e.value.message + + site_config.config['journalist_alert_email'] = 'bademail' + with pytest.raises( + securedrop_admin.JournalistAlertEmailException) as e: + site_config.validate_journalist_alert_email() + assert 'Must contain a @' in e.value.message + + site_config.config['journalist_alert_email'] = 'good@email.com' + assert site_config.validate_journalist_alert_email() + @mock.patch('securedrop_admin.SiteConfig.validated_input', side_effect=lambda p, d, v, t: d) @mock.patch('securedrop_admin.SiteConfig.save') From 7bec4edc1db449f61fb93ee4b23649b3bd2b32a0 Mon Sep 17 00:00:00 2001 From: Conor Schaefer Date: Wed, 11 Apr 2018 13:35:02 -0700 Subject: [PATCH 19/19] Small typo fixes in journalist notification docs Caught during review, didn't want to trouble @dachary by requesting changes. --- docs/install.rst | 2 +- securedrop/manage.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index a9cd8b8866..9ee736701d 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -60,7 +60,7 @@ continuing: can add more later) - The username of the system admin -You can also, optionally, configure a :doc:`daily notifications +You can also, optionally, configure :doc:`daily notifications ` about whether or not submission activity occurred in the past 24 hours. They are sent via email so journalists know if it is worth checking the *Journalist Interface*. For this you will need: diff --git a/securedrop/manage.py b/securedrop/manage.py index 79a80fb5f2..599783c7e3 100755 --- a/securedrop/manage.py +++ b/securedrop/manage.py @@ -331,7 +331,7 @@ def set_were_there_submissions_today(subps): parser = subps.add_parser( 'were-there-submissions-today', help=('Update the file indicating ' - 'iff submissions were received in the past 24h')) + 'whether submissions were received in the past 24h')) parser.set_defaults(func=were_there_submissions_today)