ossec: send journalist about the number of submissions in the past 24h

A cron job runs daily on the app server and updates the

   /var/lib/securedrop/submissions_today.txt

file which contains the number of submissions sent in the past 24h, as
created by the manage.py how_many_submissions_today command.

The OSSEC agence on the app server runs a command daily, displaying
the content of /var/lib/securedrop/submissions_today.txt. The output
of the command is sent to the OSSEC server.

A new rule is defined on the OSSEC server to send a mail to
journalist@localhost when the output is received from the OSSEC agent
running on the app server.

A new procmail rule is definied on the OSSEC server to encrypt mails
received by journalist@localhost and send them to the email defined by
the journalist_alert_email ansible variable.

A new set of ansible (optional) variables, similar to
ossec_alert_gpg_public_key, ossec_gpg_fpr, ossec_alert_email are
defined: journalist_alert_gpg_public_key, journalist_gpg_fpr,
journalist_alert_email. They are used to upload a journalist public
key to the OSSEC server and inserted into the send_encrypted_alarm.sh
script which handles mails received by procmail.

The modified send_encrypted_alarm.sh script takes one
argument (journalist or ossec) and dispatching the mail read from
stdin to the corresponding recipient.

Integration tests are implemented to verify the following:

  * manage.py how_many_submissions_today
  * the app OSSEC agent sends a mail to the journalist address
  * cover all branches of send_encrypted_alarm.sh

install_files/ansible-base/prod-specific.yml

@@ -34,6 +34,9 @@ securedrop_app_gpg_fingerprint: ""
 ossec_alert_gpg_public_key: ""
 ossec_gpg_fpr: ""
 ossec_alert_email: ""
+journalist_alert_gpg_public_key: ""
+journalist_gpg_fpr: ""
+journalist_alert_email: ""
 smtp_relay: ""
 smtp_relay_port: ""
 sasl_username: ""

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/files/cron.daily

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+cd /var/www/securedrop || exit 1
+./manage.py how-many-submissions-today

install_files/ansible-base/roles/build-securedrop-app-code-deb-pkg/tasks/build_securedrop_app_code_deb.yml

@@ -54,6 +54,17 @@
   with_items: "{{ apparmor_profiles }}"
   tags: apparmor
 
+- name: Create cron.daily directory.
+  file:
+    state: directory
+    dest: "{{ securedrop_app_code_deb_dir }}/etc/cron.daily"
+
+- name: Copy crontab.
+  copy:
+    src: cron.daily
+    dest: "{{ securedrop_app_code_deb_dir }}/etc/cron.daily/securedrop"
+    mode: 0755
+
 - name: Build securedrop-app-code Debian package.
   command: dpkg-deb --build {{ securedrop_app_code_deb_dir }}
 

install_files/ansible-base/roles/ossec/tasks/configure_server.yml

@@ -11,8 +11,12 @@
 
 - name: Copy the OSSEC GPG public key for sending encrypted alerts.
   copy:
-    src: "{{ ossec_alert_gpg_public_key }}"
+    src: "{{ item }}"
     dest: /var/ossec
+  when: "'{{ item }}' != ''"
+  with_items:
+    - "{{ ossec_alert_gpg_public_key }}"
+    - "{{ journalist_alert_gpg_public_key }}"
   tags:
     - gpg
 
@@ -21,11 +25,13 @@
   command: >
     gpg
     --homedir /var/ossec/.gnupg
-    --import /var/ossec/{{ ossec_alert_gpg_public_key }}
+    --import /var/ossec/{{ item }}
   become: yes
   become_user: "{{ ossec_group }}"
-  register: add_ossec_gpg_key_result
-  changed_when: "'imported: 1' in add_ossec_gpg_key_result.stderr"
+  when: "'{{ item }}' != ''"
+  with_items:
+    - "{{ ossec_alert_gpg_public_key }}"
+    - "{{ journalist_alert_gpg_public_key }}"
   tags:
     - gpg
 

install_files/ansible-base/roles/ossec/templates/send_encrypted_alarm.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# shellcheck disable=SC2086
 # Handler script to encrypt OSSEC alerts prior to mailing.
 # Called via the `.procmailrc` for user `ossec`.
 
@@ -16,34 +17,53 @@ set -o pipefail
 # encryption fail.
 ossec_alert_text="$(< /dev/stdin)"
 
+# default to environment even if null
+env | grep -q JOURNALIST_EMAIL || JOURNALIST_EMAIL='{{ journalist_alert_email }}'
+env | grep -q OSSEC_EMAIL || OSSEC_EMAIL='{{ ossec_alert_email }}'
+
 # Primary "send email to Admin" functionality.
 function send_encrypted_alert() {
+    local recipient="$1"
+    local gpg_fpr
+    local alert_email
+    if [[ "$recipient" = "journalist" ]] ; then
+        gpg_fpr='{{ journalist_gpg_fpr }}'
+        alert_email="$JOURNALIST_EMAIL"
+        if [[ "$alert_email" = "" ]] ; then
+            echo "journalist alert email unset, no notification sent"
+            return
+        fi
+    else
+        gpg_fpr='{{ ossec_gpg_fpr }}'
+        alert_email="$OSSEC_EMAIL"
+    fi
 
     local encrypted_alert_text
     # Try to encrypt the alert message. We'll inspect the exit status of the
     # pipeline to decide whether to send the alert text, or the default
     # failure message.
     encrypted_alert_text="$(printf "%s" "${ossec_alert_text}" | \
         /usr/bin/formail -I '' | \
-        /usr/bin/gpg --homedir /var/ossec/.gnupg --trust-model always -ear '{{ ossec_gpg_fpr }}')"
+        /usr/bin/gpg --homedir /var/ossec/.gnupg --trust-model always -ear $gpg_fpr)"
 
     # Error handling.
     if [[ -z "${encrypted_alert_text}" || $? -ne 0 ]]; then
-        send_plaintext_fail_message
+        send_plaintext_fail_message "$alert_email"
     else
         echo "${encrypted_alert_text}" | \
-            /usr/bin/mail -s "$(echo "${SUBJECT}" | sed -r 's/([0-9]{1,3}\.){3}[0-9]{1,3}\s?//g' )" '{{ ossec_alert_email }}'
+            /usr/bin/mail -s "$(echo "${SUBJECT}" | sed -r 's/([0-9]{1,3}\.){3}[0-9]{1,3}\s?//g' )" "$alert_email"
     fi
 }
 
 # Failover alerting function, in case the primary function failed.
 # Usually a failure is related to GPG balking on the encryption step;
 # that may be due to a missing pubkey or something reason.
 function send_plaintext_fail_message() {
+    local alert_email="$1"
     printf "Failed to encrypt OSSEC alert. Investigate the mailing configuration on the Monitor Server." | \
         /usr/bin/formail -I "" | \
-        /usr/bin/mail -s "$(echo "${SUBJECT}" | sed -r 's/([0-9]{1,3}\.){3}[0-9]{1,3}\s?//g' )" '{{ ossec_alert_email }}'
+        /usr/bin/mail -s "$(echo "${SUBJECT}" | sed -r 's/([0-9]{1,3}\.){3}[0-9]{1,3}\s?//g' )" "$alert_email"
 }
 
 # Encrypt the OSSEC notification and pass to mailer for sending.
-send_encrypted_alert
+send_encrypted_alert "$@"

install_files/ansible-base/roles/postfix/files/aliases

@@ -1 +1,2 @@
 root: ossec
+journalist: ossec

install_files/ansible-base/roles/postfix/files/procmailrc

@@ -5,4 +5,8 @@ LOGFILE=/var/log/procmail.log
 SUBJECT=`formail -xSubject:`
 :0 c
 *^To:.*root.*
-|/var/ossec/send_encrypted_alarm.sh
+|/var/ossec/send_encrypted_alarm.sh ossec
+
+:0 c
+*^To:.*journalist.*
+|/var/ossec/send_encrypted_alarm.sh journalist

install_files/securedrop-ossec-agent/var/ossec/etc/ossec.conf

@@ -34,6 +34,8 @@
 
     <ignore>/var/lib/securedrop/db.sqlite</ignore>
 
+    <ignore>/var/lib/securedrop/submissions_today.txt</ignore>
+
     <ignore>/var/securedrop/store</ignore>
 
     <ignore>/var/ossec/queue</ignore>
@@ -101,6 +103,12 @@
     <command>last -n 5</command>
   </localfile>
 
+  <localfile>
+    <log_format>full_command</log_format>
+    <command>head -1 /var/lib/securedrop/submissions_today.txt | grep '^[0-9]*$'</command>
+    <frequency>86400</frequency>
+  </localfile>
+
   <localfile>
     <log_format>syslog</log_format>
     <location>/var/log/kern.log</location>

install_files/securedrop-ossec-server/var/ossec/etc/ossec.conf

@@ -124,6 +124,12 @@
     <do_not_delay />
   </email_alerts>
 
+  <email_alerts>
+    <email_to>journalist@localhost</email_to>
+    <group>multiple_drops</group>
+    <do_not_delay />
+  </email_alerts>
+
   <email_alerts>
     <email_to>root@localhost</email_to>
     <group>low_diskspace</group>

install_files/securedrop-ossec-server/var/ossec/rules/local_rules.xml

@@ -190,3 +190,11 @@
     <options>no_email_alert</options>
   </rule>
 </group>
+
+<group name="multiple_drops">
+  <rule id="400600" level="7" >
+    <if_sid>530</if_sid>
+    <match>ossec: output: 'head -1 /var/lib/securedrop/submissions_today.txt</match>
+    <description>Number of submissions received in the past 24h.</description>
+  </rule>
+</group>

testinfra/ossec/test_journalist_mail.py

@@ -0,0 +1,208 @@
+import pytest
+import testinfra
+import time
+
+
+class TestBase(object):
+
+    @pytest.fixture(autouse=True)
+    def only_mon_staging_sudo(self, host):
+        if host.backend.host != 'mon-staging':
+            pytest.skip()
+
+        with host.sudo():
+            yield
+
+    def ansible(self, host, module, parameters):
+        r = host.ansible(module, parameters, check=False)
+        assert 'exception' not in r
+
+    def run(self, host, cmd):
+        print(host.backend.host + " running: " + cmd)
+        r = host.run(cmd)
+        print(r.stdout)
+        print(r.stderr)
+        return r.rc == 0
+
+    def wait_for(self, fun):
+        success = False
+        for d in (1, 2, 4, 8, 16, 32, 64):
+            if fun():
+                success = True
+                break
+            time.sleep(d)
+        return success
+
+    def wait_for_command(self, host, cmd):
+        return self.wait_for(lambda: self.run(host, cmd))
+
+    #
+    # implementation note: we do not use host.ansible("service", ...
+    # because it only works for services in /etc/init and not those
+    # legacy only found in /etc/init.d such as postfix
+    #
+    def service_started(self, host, name):
+        assert self.run(host, "service {name} start".format(name=name))
+        assert self.wait_for_command(
+            host,
+            "service {name} status | grep -q 'is running'".format(name=name))
+
+    def service_restarted(self, host, name):
+        assert self.run(host, "service {name} restart".format(name=name))
+        assert self.wait_for_command(
+            host,
+            "service {name} status | grep -q 'is running'".format(name=name))
+
+    def service_stopped(self, host, name):
+        assert self.run(host, "service {name} stop".format(name=name))
+        assert self.wait_for_command(
+            host,
+            "service {name} status | grep -q 'not running'".format(name=name))
+
+
+class TestJournalistMail(TestBase):
+
+    def test_procmail(self, host):
+        self.service_started(host, "postfix")
+        for (origin, destination) in (
+                ('journalist', 'journalist'),
+                ('root', 'ossec')):
+            assert self.run(host, "postsuper -d ALL")
+            assert self.run(
+                host,
+                "echo DEF | mail -s 'abc' {origin}@localhost".format(
+                    origin=origin))
+            assert self.wait_for_command(
+                host,
+                "mailq | grep -q {destination}@ossec.test".format(
+                    destination=destination))
+        self.service_stopped(host, "postfix")
+
+    def test_send_encrypted_alert(self, host):
+        self.service_started(host, "postfix")
+        src = "install_files/ansible-base/roles/ossec/files/test_admin_key.sec"
+        self.ansible(host, "copy",
+                     "dest=/tmp/test_admin_key.sec src={src}".format(src=src))
+
+        self.run(host, "gpg  --homedir /var/ossec/.gnupg"
+                 " --import /tmp/test_admin_key.sec")
+
+        def trigger(who):
+            assert self.run(
+                host, "! mailq | grep -q {who}@ossec.test".format(who=who))
+            assert self.run(
+                host,
+                """
+                ( echo 'Subject: TEST' ; echo ; echo MYGREATPAYLOAD ) | \
+                /var/ossec/send_encrypted_alarm.sh {who}
+                """.format(who=who))
+            assert self.wait_for_command(
+                host, "mailq | grep -q {who}@ossec.test".format(who=who))
+
+        #
+        # encrypted mail to journalist or ossec contact
+        #
+        for who in ('journalist', 'ossec'):
+            assert self.run(host, "postsuper -d ALL")
+            trigger(who)
+            assert self.run(
+                host,
+                """
+                job=$(mailq | sed -n -e '2p' | cut -f1 -d ' ')
+                postcat -q $job | tee /dev/stderr | \
+                   gpg --homedir /var/ossec/.gnupg --decrypt 2>&1 | \
+                   grep -q MYGREATPAYLOAD
+                """)
+        #
+        # failure to encrypt must trigger an emergency mail to ossec contact
+        #
+        try:
+            assert self.run(host, "postsuper -d ALL")
+            assert self.run(host, "mv /usr/bin/gpg /usr/bin/gpg.save")
+            trigger(who)
+            assert self.run(
+                host,
+                """
+                job=$(mailq | sed -n -e '2p' | cut -f1 -d ' ')
+                postcat -q $job | grep -q 'Failed to encrypt OSSEC alert'
+                """)
+        finally:
+            assert self.run(host, "mv /usr/bin/gpg.save /usr/bin/gpg")
+        self.service_stopped(host, "postfix")
+
+    def test_missing_journalist_alert(self, host):
+        #
+        # missing journalist mail does nothing
+        #
+        assert self.run(
+            host,
+            """
+            JOURNALIST_EMAIL= \
+               bash -x /var/ossec/send_encrypted_alarm.sh journalist | \
+               tee /dev/stderr | \
+               grep -q 'no notification sent'
+            """)
+
+    # https://ossec-docs.readthedocs.io/en/latest/manual/rules-decoders/testing.html
+    def test_ossec_rule_journalist(self, host):
+        assert self.run(host, """
+        set -ex
+        l="ossec: output: 'head -1 /var/lib/securedrop/submissions_today.txt"
+        echo "$l" | /var/ossec/bin/ossec-logtest
+        echo "$l" | /var/ossec/bin/ossec-logtest -U '400600:7:ossec'
+        """)
+
+    def test_journalist_mail_notification(self, host):
+        mon = host
+        app = testinfra.host.Host.get_host(
+            'ansible://app-staging',
+            ansible_inventory=host.backend.ansible_inventory)
+        #
+        # run ossec & postfix on mon
+        #
+        self.service_started(mon, "postfix")
+        self.service_started(mon, "ossec")
+
+        #
+        # ensure the submission_today.txt file exists
+        #
+        with app.sudo():
+            assert self.run(app, """
+            cd /var/www/securedrop
+            ./manage.py how-many-submissions-today
+            test -f /var/lib/securedrop/submissions_today.txt
+            """)
+
+        #
+        # empty the mailq on mon in case there were leftovers
+        #
+        assert self.run(mon, "postsuper -d ALL")
+
+        #
+        # start ossec with frequent monitoring of
+        # submissions_today.txt
+        #
+        with app.sudo():
+            assert self.run(
+                app,
+                "sed -i -e 's/>86400</>7</' /var/ossec/etc/ossec.conf")
+            self.service_restarted(app, "ossec")
+
+        #
+        # wait until at least one notification is sent
+        #
+        assert self.wait_for_command(
+            mon,
+            "mailq | grep -q [email protected]")
+
+        #
+        # teardown the ossec and postfix on mon and app
+        #
+        self.service_stopped(mon, "postfix")
+        self.service_stopped(mon, "ossec")
+
+        with app.sudo():
+            self.service_stopped(app, "ossec")
+            assert self.run(
+                app,
+                "sed -i -e 's/>7</>86400</' /var/ossec/etc/ossec.conf")