Merge pull request #6892 from freedomofpress/sequoia-initial

New source creation uses Sequoia

install_files/ansible-base/roles/app/tasks/initialize_securedrop_app.yml

@@ -1,18 +1,24 @@
 ---
 - name: Copy the SecureDrop Application GPG public key to the Application Server.
+  # The key is installed as journalist.pub for Sequoia to read. Since this controls
+  # who can decrypt things, set it to root-writable, but www-data-readable.
   copy:
     src: "{{ securedrop_app_gpg_public_key }}"
-    dest: "{{ securedrop_data }}/"
+    dest: "{{ securedrop_data }}/journalist.pub"
+    owner: "root"
+    group: "www-data"
+    mode: "0640"
   tags:
     - securedrop_config
 
 - name: Import the SecureDrop Application GPG public key to the Application Server keyring.
-  # multiline format for command module, since this is a long command
+  # TODO: We don't need this but it sets up the GPG keyring so let's leave it in place now
+  # and get rid of it once we've finished the private key migration step to Sequoia.
   command: >
     su -s /bin/bash -c 'gpg
     --homedir {{ securedrop_data }}/keys
     --no-default-keyring --keyring {{ securedrop_data }}/keys/pubring.gpg
-    --import {{ securedrop_data }}/{{ securedrop_app_gpg_public_key }}' {{ securedrop_user }}
+    --import {{ securedrop_data }}/journalist.pub' {{ securedrop_user }}
   register: gpg_app_key_import
   changed_when: "'imported: 1' in gpg_app_key_import.stderr"
   tags:

molecule/testinfra/app-code/test_securedrop_app_code.py

@@ -72,15 +72,15 @@ def test_securedrop_application_test_journalist_key(host):
     Ensure the SecureDrop Application GPG public key file is present.
     This is a test-only pubkey provided in the repository strictly for testing.
     """
-    pubkey_file = host.file(f"{securedrop_test_vars.securedrop_data}/test_journalist_key.pub")
+    pubkey_file = host.file(f"{securedrop_test_vars.securedrop_data}/journalist.pub")
     # sudo is only necessary when testing against app hosts, since the
     # permissions are tighter. Let's elevate privileges so we're sure
     # we can read the correct file attributes and test them.
     with host.sudo():
         assert pubkey_file.is_file
         assert pubkey_file.user == "root"
-        assert pubkey_file.group == "root"
-        assert pubkey_file.mode == 0o644
+        assert pubkey_file.group == "www-data"
+        assert pubkey_file.mode == 0o640
 
     # Let's make sure the corresponding fingerprint is specified
     # in the SecureDrop app configuration.

molecule/testinfra/app/test_smoke.py

@@ -11,14 +11,15 @@
 
 
 @pytest.mark.parametrize(
-    ("name", "url", "curl_flags"),
+    ("name", "url", "curl_flags", "expected"),
     [
         # We pass -L to follow the redirect from / to /login
-        ("journalist", "http://localhost:8080/", "L"),
-        ("source", "http://localhost:80/", ""),
+        ("journalist", "http://localhost:8080/", "L", "Powered by"),
+        ("source", "http://localhost:80/", "", "Powered by"),
+        ("source", "http://localhost:80/public-key", "", "-----BEGIN PGP PUBLIC KEY BLOCK-----"),
     ],
 )
-def test_interface_up(host, name, url, curl_flags):
+def test_interface_up(host, name, url, curl_flags, expected):
     """
     Ensure the respective interface is up with HTTP 200 if not, we try our
     best to grab the error log and print it via an intentionally failed
@@ -32,7 +33,7 @@ def test_interface_up(host, name, url, curl_flags):
             if f.exists:
                 assert "nopenopenope" in f.content_string
     assert "200 OK" in response
-    assert "Powered by" in response
+    assert expected in response
 
 
 def test_redwood(host):

redwood/redwood/__init__.pyi

@@ -1,8 +1,11 @@
 # type stub for redwood module
 # see https://pyo3.rs/v0.16.4/python_typing_hints.html
 from pathlib import Path
+from typing import BinaryIO
 
 def generate_source_key_pair(passphrase: str, email: str) -> (str, str, str): ...
 def encrypt_message(recipients: list[str], plaintext: str, destination: Path) -> None: ...
-def encrypt_file(recipients: list[str], plaintext: Path, destination: Path) -> None: ...
-def decrypt(ciphertext: bytes, secret_key: str, passphrase: str) -> str: ...
+def encrypt_stream(recipients: list[str], plaintext: BinaryIO, destination: Path) -> None: ...
+def decrypt(ciphertext: bytes, secret_key: str, passphrase: str) -> bytes: ...
+
+class RedwoodError(Exception): ...

redwood/src/lib.rs

@@ -12,17 +12,16 @@ use sequoia_openpgp::serialize::{
     SerializeInto,
 };
 use sequoia_openpgp::Cert;
+use std::borrow::Cow;
 use std::fs::File;
 use std::io::{self, Read};
 use std::path::{Path, PathBuf};
 use std::str::FromStr;
 use std::string::FromUtf8Error;
 use std::time::{Duration, SystemTime};
 
-/// Alias to make it easier for Python readers
-type Bytes = Vec<u8>;
-
 mod decryption;
+mod stream;
 
 #[derive(thiserror::Error, Debug)]
 pub enum Error {
@@ -60,7 +59,7 @@ const KEY_CREATION_SECONDS_FROM_EPOCH: u64 = 1368507600;
 fn redwood(py: Python, m: &PyModule) -> PyResult<()> {
     m.add_function(wrap_pyfunction!(generate_source_key_pair, m)?)?;
     m.add_function(wrap_pyfunction!(encrypt_message, m)?)?;
-    m.add_function(wrap_pyfunction!(encrypt_file, m)?)?;
+    m.add_function(wrap_pyfunction!(encrypt_stream, m)?)?;
     m.add_function(wrap_pyfunction!(decrypt, m)?)?;
     m.add("RedwoodError", py.get_type::<RedwoodError>())?;
     Ok(())
@@ -107,17 +106,17 @@ pub fn encrypt_message(
     encrypt(&recipients, plaintext, &destination)
 }
 
-/// Encrypt a file that's already on disk for the specified recipients.
+/// Encrypt a Python stream (`typing.BinaryIO`) for the specified recipients.
 /// The list of recipients is a set of PGP public keys. The encrypted file
 /// will be written to `destination`.
 #[pyfunction]
-pub fn encrypt_file(
+pub fn encrypt_stream(
     recipients: Vec<String>,
-    plaintext: PathBuf,
+    plaintext: &PyAny,
     destination: PathBuf,
 ) -> Result<()> {
-    let plaintext = File::open(plaintext)?;
-    encrypt(&recipients, plaintext, &destination)
+    let stream = stream::Stream { reader: plaintext };
+    encrypt(&recipients, stream, &destination)
 }
 
 /// Helper function to encrypt readable things.
@@ -175,14 +174,14 @@ fn encrypt(
 }
 
 /// Given a ciphertext, private key, and passphrase, unlock the private key with
-/// the passphrase, and use it to decrypt the ciphertext.  It is assumed that the
-/// plaintext is UTF-8.
+/// the passphrase, and use it to decrypt the ciphertext. Arbitrary bytes are
+/// returned, which may or may not be valid UTF-8.
 #[pyfunction]
 pub fn decrypt(
-    ciphertext: Bytes,
+    ciphertext: Vec<u8>,
     secret_key: String,
     passphrase: String,
-) -> Result<String> {
+) -> Result<Cow<'static, [u8]>> {
     let recipient = Cert::from_str(&secret_key)?;
     let policy = &StandardPolicy::new();
     let passphrase: Password = passphrase.into();
@@ -197,10 +196,10 @@ pub fn decrypt(
         .with_policy(policy, None, helper)?;
 
     // Decrypt the data.
-    let mut buffer: Bytes = vec![];
+    let mut buffer: Vec<u8> = vec![];
     io::copy(&mut decryptor, &mut buffer)?;
-    let plaintext = String::from_utf8(buffer)?;
-    Ok(plaintext)
+    // pyo3 maps Cow<[u8]> to Python's bytes
+    Ok(Cow::from(buffer))
 }
 
 #[cfg(test)]
@@ -315,7 +314,10 @@ mod tests {
         )
         .unwrap();
         // Verify message is what we put in originally
-        assert_eq!(SECRET_MESSAGE, &plaintext);
+        assert_eq!(
+            SECRET_MESSAGE,
+            String::from_utf8(plaintext.to_vec()).unwrap()
+        );
         // Decrypt as key 2
         let plaintext = decrypt(
             ciphertext.clone().into_bytes(),
@@ -324,7 +326,10 @@ mod tests {
         )
         .unwrap();
         // Verify message is what we put in originally
-        assert_eq!(SECRET_MESSAGE, &plaintext);
+        assert_eq!(
+            SECRET_MESSAGE,
+            String::from_utf8(plaintext.to_vec()).unwrap()
+        );
         // Try to decrypt as key 3, expect an error
         let err = decrypt(
             ciphertext.into_bytes(),

redwood/src/stream.rs

@@ -0,0 +1,32 @@
+use pyo3::types::PyBytes;
+use pyo3::{PyAny, PyResult};
+use std::io::{self, ErrorKind, Read, Write};
+
+/// Wrapper to implement the `Read` trait around a Python
+/// object that contains a `.read()` function.
+pub(crate) struct Stream<'a> {
+    pub(crate) reader: &'a PyAny,
+}
+
+impl Stream<'_> {
+    /// Read the specified number of bytes out of the object
+    fn read_bytes(&self, len: usize) -> PyResult<&PyBytes> {
+        let func = self.reader.getattr("read")?;
+        // In Python this is effectively calling `reader.read(len)`
+        let bytes = func.call1((len,))?;
+        let bytes = bytes.downcast::<PyBytes>()?;
+        Ok(bytes)
+    }
+}
+
+impl Read for Stream<'_> {
+    fn read(&mut self, mut buf: &mut [u8]) -> std::io::Result<usize> {
+        let bytes = self.read_bytes(buf.len()).map_err(|err| {
+            // The PyErr could be a type error (e.g. no "read" method) or an
+            // actual I/O failure if the read() call failed, let's just treat
+            // all of them as "other" for simplicity.
+            io::Error::new(ErrorKind::Other, err.to_string())
+        })?;
+        buf.write(bytes.as_bytes())
+    }
+}

securedrop/alembic/versions/811334d7105f_sequoia_pgp.py

@@ -0,0 +1,30 @@
+"""sequoia_pgp
+
+Revision ID: 811334d7105f
+Revises: c5a02eb52f2d
+Create Date: 2023-06-29 18:19:59.314380
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = "811334d7105f"
+down_revision = "c5a02eb52f2d"
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+    with op.batch_alter_table("sources", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("pgp_fingerprint", sa.String(length=40), nullable=True))
+        batch_op.add_column(sa.Column("pgp_public_key", sa.Text(), nullable=True))
+        batch_op.add_column(sa.Column("pgp_secret_key", sa.Text(), nullable=True))
+
+
+def downgrade() -> None:
+    # We do NOT drop the columns here, because doing so would break any
+    # source that had its key pair stored here. If a downgrade is needed for
+    # whatever reason, the extra columns will just be ignored, and the sources
+    # will still be temporarily broken, but there will be no data loss.
+    pass

securedrop/bin/dev-deps

@@ -112,8 +112,8 @@ function reset_demo() {
     # Set up GPG keys directory structure.
     sudo mkdir -p /var/lib/securedrop/{store,keys,tmp}
     sudo chown -R "$(id -u)" /var/lib/securedrop
-    cp ./tests/files/test_journalist_key.pub /var/lib/securedrop/keys
-    gpg2 --homedir /var/lib/securedrop/keys --import /var/lib/securedrop/keys/test_journalist_key.pub >& /tmp/gpg.out || cat /tmp/gpg.out
+    cp ./tests/files/test_journalist_key.pub /var/lib/securedrop/journalist.pub
+    gpg2 --homedir /var/lib/securedrop/keys --import /var/lib/securedrop/journalist.pub >& /tmp/gpg.out || cat /tmp/gpg.out
 
     # Create gpg-agent.conf
     echo allow-loopback-pinentry > /var/lib/securedrop/keys/gpg-agent.conf
@@ -144,7 +144,7 @@ function reset_demo() {
 
     ./manage.py reset
 
-    gpg2 --homedir /var/lib/securedrop/keys --no-default-keyring --keyring /var/lib/securedrop/keys/pubring.gpg --import /var/lib/securedrop/keys/test_journalist_key.pub
+    gpg2 --homedir /var/lib/securedrop/keys --no-default-keyring --keyring /var/lib/securedrop/keys/pubring.gpg --import /var/lib/securedrop/journalist.pub
 
     # Can't pass an array environment variable with "docker --env", so
     # break up the string we can pass.

securedrop/debian/app-code/etc/apparmor.d/usr.sbin.apache2

@@ -120,6 +120,7 @@
   /var/lib/securedrop/db.sqlite rwk,
   /var/lib/securedrop/db.sqlite-journal rw,
   /var/lib/securedrop/db.sqlite-journal w,
+  /var/lib/securedrop/journalist.pub r,
   /var/lib/securedrop/keys/* rwl,
   /var/lib/securedrop/keys/*.app-staging.* w,
   /var/lib/securedrop/keys/gpg-agent.conf r,

securedrop/debian/securedrop-app-code.postinst

@@ -168,6 +168,31 @@ update_apache2_headers(){
     sed -i 's/^Header always set Cross-Origin-Resource-Policy "same-site"/Header always set Cross-Origin-Resource-Policy "same-origin"/g' "$1"
 }
 
+# Export the journalist public key out of the GPG keyring to journalist.pub
+# as part of the GPG -> Sequoia migration.
+export_journalist_public_key() {
+    # config.py is root-writable, so it's safe to run it as root to extract the fingerprint
+    journalist_pub="/var/lib/securedrop/journalist.pub"
+    # If the journalist.pub file doesn't exist
+    if ! test -f $journalist_pub; then
+        # And we have a config.py (during initial install it won't exist yet)
+        if test -f /var/www/securedrop/config.py; then
+            # n.b. based on sdconfig.py, this should work with very old config.py files.
+            fingerprint=$(cd /var/www/securedrop; python3 -c "import config; print(config.JOURNALIST_KEY)")
+            # Set up journalist.pub as root/www-data 640 before writing to it.
+            touch $journalist_pub
+            chown root:www-data $journalist_pub
+            chmod 640 $journalist_pub
+            # Export the GPG public key
+            # shellcheck disable=SC2024
+            sudo -u www-data gpg2 --homedir=/var/lib/securedrop/keys --export --armor "$fingerprint" > $journalist_pub
+            # Verify integrity of what we just exported
+            sudo -u www-data /var/www/securedrop/scripts/validate-pgp-key "$journalist_pub" "$fingerprint"
+        fi
+    fi
+
+}
+
 
 case "$1" in
     configure)
@@ -232,10 +257,11 @@ case "$1" in
     # mode.
     a2dissite 000-default
     a2dissite default-ssl
-    # Stop Apache service before making changes to its AppArmor profile.
-    # If the Apache service is running unconfined, and the profile is
+    # Stop Apache service while we make changes. This is primarily for
+    # AppArmor (if the Apache service is running unconfined, and the profile is
     # set to "enforce", then apache2 will fail to restart, since it lacks
-    # the ability to send signals to unconfined peers.
+    # the ability to send signals to unconfined peers), but also prevents any
+    # web traffic while database updates are applied.
     service apache2 stop
 
     # and make sure it's enabled
@@ -259,17 +285,20 @@ case "$1" in
     update_apache2_headers /etc/apache2/sites-available/source.conf
     update_apache2_headers /etc/apache2/sites-available/journalist.conf
 
-    # Restart apache so it loads with the apparmor profiles in enforce mode.
-    service apache2 restart
-
     # remove previously dynamically-generated assets
     rm -fr /var/www/securedrop/static/gen/
     rm -fr /var/www/securedrop/static/.webassets-cache/
 
-    # Version migrations
+    # GPG -> Sequoia migration
+    export_journalist_public_key
 
+    # Version migrations
     database_migration
 
+    # Restart apache now that we've updated everything, setup AppArmor
+    # and applied all migrations
+    service apache2 restart
+
     ;;
 
     abort-upgrade|abort-remove|abort-deconfigure|triggered)