Argon2 the second part: implement key encryption / decryption (#6469)

Argon2 the second part: implement encryption/decryption of argon2 keys

borg init --key-algorithm=argon2 (new default, older pbkdf2 also still available)

borg key change-passphrase: keep key algorithm the same
borg key change-location: keep key algorithm the same

use env var BORG_TESTONLY_WEAKEN_KDF=1 to resource limit (cpu, memory, ...) the kdf when running the automated tests.

conftest.py

@@ -2,14 +2,6 @@
 
 import pytest
 
-# IMPORTANT keep this above all other borg imports to avoid inconsistent values
-# for `from borg.constants import PBKDF2_ITERATIONS` (or star import) usages before
-# this is executed
-from borg import constants
-# no fixture-based monkey-patching since star-imports are used for the constants module
-constants.PBKDF2_ITERATIONS = 1
-
-
 # needed to get pretty assertion failures in unit tests:
 if hasattr(pytest, 'register_assert_rewrite'):
     pytest.register_assert_rewrite('borg.testsuite')
@@ -36,6 +28,8 @@ def clean_env(tmpdir_factory, monkeypatch):
             if key.startswith('BORG_') and key not in ('BORG_FUSE_IMPL', )]
     for key in keys:
         monkeypatch.delenv(key, raising=False)
+    # Speed up tests
+    monkeypatch.setenv("BORG_TESTONLY_WEAKEN_KDF", "1")
 
 
 def pytest_report_header(config, startdir):

src/borg/archiver.py

@@ -398,7 +398,8 @@ def do_change_location(self, args, repository, manifest, key, cache):
             setattr(key_new, name, value)
 
         key_new.target = key_new.get_new_target(args)
-        key_new.save(key_new.target, key._passphrase, create=True)  # save with same passphrase
+        # save with same passphrase and algorithm
+        key_new.save(key_new.target, key._passphrase, create=True, algorithm=key._encrypted_key_algorithm)
 
         # rewrite the manifest with the new key, so that the key-type byte of the manifest changes
         manifest.key = key_new
@@ -4323,6 +4324,7 @@ def define_borg_mount(parser):
                                help='Set storage quota of the new repository (e.g. 5G, 1.5T). Default: no quota.')
         subparser.add_argument('--make-parent-dirs', dest='make_parent_dirs', action='store_true',
                                help='create the parent directories of the repository directory, if they are missing.')
+        subparser.add_argument('--key-algorithm', dest='key_algorithm', default='argon2', choices=list(KEY_ALGORITHMS))
 
         # borg key
         subparser = subparsers.add_parser('key', parents=[mid_common_parser], add_help=False,

src/borg/constants.py

@@ -103,6 +103,18 @@
 
 PBKDF2_ITERATIONS = 100000
 
+# https://www.rfc-editor.org/rfc/rfc9106.html#section-4-6.2
+ARGON2_ARGS = {'time_cost': 3, 'memory_cost': 2**16, 'parallelism': 4, 'type': 'id'}
+ARGON2_SALT_BYTES = 16
+
+# Maps the CLI argument to our internal identifier for the format
+KEY_ALGORITHMS = {
+    # encrypt-and-MAC, kdf: PBKDF2(HMAC−SHA256), encryption: AES256-CTR, authentication: HMAC-SHA256
+    'pbkdf2': 'sha256',
+    # encrypt-then-MAC, kdf: argon2, encryption: AES256-CTR, authentication: HMAC-SHA256
+    'argon2': 'argon2 aes256-ctr hmac-sha256',
+}
+
 
 class KeyBlobStorage:
     NO_STORAGE = 'no_storage'

src/borg/crypto/key.py

@@ -25,6 +25,7 @@
 from .nonces import NonceManager
 from .low_level import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256, hkdf_hmac_sha512
 from .low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b, AES256_OCB, CHACHA20_POLY1305
+from . import low_level
 
 
 class UnsupportedPayloadError(Error):
@@ -51,6 +52,10 @@ class RepoKeyNotFoundError(Error):
     """No key entry found in the config of repository {}."""
 
 
+class UnsupportedKeyFormatError(Error):
+    """Your borg key is stored in an unsupported format. Try using a newer version of borg."""
+
+
 class TAMRequiredError(IntegrityError):
     __doc__ = textwrap.dedent("""
     Manifest is unauthenticated, but it is required for this repository.
@@ -430,15 +435,54 @@ def decrypt_key_file(self, data, passphrase):
         unpacker = get_limited_unpacker('key')
         unpacker.feed(data)
         data = unpacker.unpack()
-        enc_key = EncryptedKey(internal_dict=data)
-        assert enc_key.version == 1
-        assert enc_key.algorithm == 'sha256'
-        key = passphrase.kdf(enc_key.salt, enc_key.iterations, 32)
-        data = AES(key, b'\0'*16).decrypt(enc_key.data)
-        if hmac.compare_digest(hmac_sha256(key, data), enc_key.hash):
+        encrypted_key = EncryptedKey(internal_dict=data)
+        if encrypted_key.version != 1:
+            raise UnsupportedKeyFormatError()
+        else:
+            self._encrypted_key_algorithm = encrypted_key.algorithm
+            if encrypted_key.algorithm == 'sha256':
+                return self.decrypt_key_file_pbkdf2(encrypted_key, passphrase)
+            elif encrypted_key.algorithm == 'argon2 aes256-ctr hmac-sha256':
+                return self.decrypt_key_file_argon2(encrypted_key, passphrase)
+            else:
+                raise UnsupportedKeyFormatError()
+
+    def decrypt_key_file_pbkdf2(self, encrypted_key, passphrase):
+        key = passphrase.kdf(encrypted_key.salt, encrypted_key.iterations, 32)
+        data = AES(key, b'\0'*16).decrypt(encrypted_key.data)
+        if hmac.compare_digest(hmac_sha256(key, data), encrypted_key.hash):
             return data
+        return None
+
+    def decrypt_key_file_argon2(self, encrypted_key, passphrase):
+        key = passphrase.argon2(
+            output_len_in_bytes=64,
+            salt=encrypted_key.salt,
+            time_cost=encrypted_key.argon2_time_cost,
+            memory_cost=encrypted_key.argon2_memory_cost,
+            parallelism=encrypted_key.argon2_parallelism,
+            type=encrypted_key.argon2_type,
+        )
+        enc_key, mac_key = key[:32], key[32:]
+        ae_cipher = AES256_CTR_HMAC_SHA256(
+            iv=0, header_len=0, aad_offset=0,
+            enc_key=enc_key,
+            mac_key=mac_key,
+        )
+        try:
+            return ae_cipher.decrypt(encrypted_key.data)
+        except low_level.IntegrityError:
+            return None
+
+    def encrypt_key_file(self, data, passphrase, algorithm):
+        if algorithm == 'sha256':
+            return self.encrypt_key_file_pbkdf2(data, passphrase)
+        elif algorithm == 'argon2 aes256-ctr hmac-sha256':
+            return self.encrypt_key_file_argon2(data, passphrase)
+        else:
+            raise ValueError(f'Unexpected algorithm: {algorithm}')
 
-    def encrypt_key_file(self, data, passphrase):
+    def encrypt_key_file_pbkdf2(self, data, passphrase):
         salt = os.urandom(32)
         iterations = PBKDF2_ITERATIONS
         key = passphrase.kdf(salt, iterations, 32)
@@ -454,7 +498,29 @@ def encrypt_key_file(self, data, passphrase):
         )
         return msgpack.packb(enc_key.as_dict())
 
-    def _save(self, passphrase):
+    def encrypt_key_file_argon2(self, data, passphrase):
+        salt = os.urandom(ARGON2_SALT_BYTES)
+        key = passphrase.argon2(
+            output_len_in_bytes=64,
+            salt=salt,
+            **ARGON2_ARGS,
+        )
+        enc_key, mac_key = key[:32], key[32:]
+        ae_cipher = AES256_CTR_HMAC_SHA256(
+            iv=0, header_len=0, aad_offset=0,
+            enc_key=enc_key,
+            mac_key=mac_key,
+        )
+        encrypted_key = EncryptedKey(
+            version=1,
+            algorithm='argon2 aes256-ctr hmac-sha256',
+            salt=salt,
+            data=ae_cipher.encrypt(data),
+            **{'argon2_' + k: v for k, v in ARGON2_ARGS.items()},
+        )
+        return msgpack.packb(encrypted_key.as_dict())
+
+    def _save(self, passphrase, algorithm):
         key = Key(
             version=1,
             repository_id=self.repository_id,
@@ -464,14 +530,14 @@ def _save(self, passphrase):
             chunk_seed=self.chunk_seed,
             tam_required=self.tam_required,
         )
-        data = self.encrypt_key_file(msgpack.packb(key.as_dict()), passphrase)
+        data = self.encrypt_key_file(msgpack.packb(key.as_dict()), passphrase, algorithm)
         key_data = '\n'.join(textwrap.wrap(b2a_base64(data).decode('ascii')))
         return key_data
 
     def change_passphrase(self, passphrase=None):
         if passphrase is None:
             passphrase = Passphrase.new(allow_empty=True)
-        self.save(self.target, passphrase)
+        self.save(self.target, passphrase, algorithm=self._encrypted_key_algorithm)
 
     @classmethod
     def create(cls, repository, args):
@@ -481,7 +547,7 @@ def create(cls, repository, args):
         key.init_from_random_data()
         key.init_ciphers()
         target = key.get_new_target(args)
-        key.save(target, passphrase, create=True)
+        key.save(target, passphrase, create=True, algorithm=KEY_ALGORITHMS[args.key_algorithm])
         logger.info('Key in "%s" created.' % target)
         logger.info('Keep this key safe. Your data will be inaccessible without it.')
         return key
@@ -581,8 +647,8 @@ def load(self, target, passphrase):
             self.target = target
         return success
 
-    def save(self, target, passphrase, create=False):
-        key_data = self._save(passphrase)
+    def save(self, target, passphrase, algorithm, create=False):
+        key_data = self._save(passphrase, algorithm)
         if self.STORAGE == KeyBlobStorage.KEYFILE:
             if create and os.path.isfile(target):
                 # if a new keyfile key repository is created, ensure that an existing keyfile of another
@@ -657,8 +723,8 @@ def load(self, target, passphrase):
         self.logically_encrypted = False
         return success
 
-    def save(self, target, passphrase, create=False):
-        super().save(target, passphrase, create=create)
+    def save(self, target, passphrase, algorithm, create=False):
+        super().save(target, passphrase, algorithm, create=create)
         self.logically_encrypted = False
 
     def init_ciphers(self, manifest_data=None):

src/borg/helpers/passphrase.py

@@ -141,6 +141,8 @@ def __repr__(self):
         return '<Passphrase "***hidden***">'
 
     def kdf(self, salt, iterations, length):
+        if os.environ.get("BORG_TESTONLY_WEAKEN_KDF") == "1":
+            iterations = 1
         return pbkdf2_hmac('sha256', self.encode('utf-8'), salt, iterations, length)
 
     def argon2(
@@ -152,6 +154,11 @@ def argon2(
         parallelism,
         type: Literal['i', 'd', 'id']
     ) -> bytes:
+        if os.environ.get("BORG_TESTONLY_WEAKEN_KDF") == "1":
+            time_cost = 1
+            parallelism = 1
+            # 8 is the smallest value that avoids the "Memory cost is too small" exception
+            memory_cost = 8
         type_map = {
             'i': argon2.low_level.Type.I,
             'd': argon2.low_level.Type.D,

src/borg/item.pyx

@@ -7,6 +7,7 @@ from .helpers import bigint_to_int, int_to_bigint
 from .helpers import StableDict
 from .helpers import format_file_size
 
+
 cdef extern from "_item.c":
     object _object_to_optr(object obj)
     object _optr_to_object(object bytes)
@@ -294,7 +295,8 @@ class EncryptedKey(PropDict):
     If a EncryptedKey shall be serialized, give as_dict() method output to msgpack packer.
     """
 
-    VALID_KEYS = {'version', 'algorithm', 'iterations', 'salt', 'hash', 'data'}  # str-typed keys
+    VALID_KEYS = { 'version', 'algorithm', 'iterations', 'salt', 'hash', 'data',
+                   'argon2_time_cost', 'argon2_memory_cost', 'argon2_parallelism', 'argon2_type' }
 
     __slots__ = ("_dict", )  # avoid setting attributes not supported by properties
 
@@ -304,6 +306,10 @@ class EncryptedKey(PropDict):
     salt = PropDict._make_property('salt', bytes)
     hash = PropDict._make_property('hash', bytes)
     data = PropDict._make_property('data', bytes)
+    argon2_time_cost = PropDict._make_property('argon2_time_cost', int)
+    argon2_memory_cost = PropDict._make_property('argon2_memory_cost', int)
+    argon2_parallelism = PropDict._make_property('argon2_parallelism', int)
+    argon2_type = PropDict._make_property('argon2_type', str, encode=str.encode, decode=bytes.decode)
 
 
 class Key(PropDict):

src/borg/testsuite/archiver.py

@@ -16,7 +16,7 @@
 import tempfile
 import time
 import unittest
-from binascii import unhexlify, b2a_base64
+from binascii import unhexlify, b2a_base64, a2b_base64
 from configparser import ConfigParser
 from datetime import datetime
 from datetime import timezone
@@ -3585,6 +3585,51 @@ def test_recovery_from_deleted_repo_nonce(self):
         self.cmd('create', self.repository_location + '::test2', 'input')
         assert os.path.exists(nonce)
 
+    def test_init_defaults_to_argon2(self):
+        """https://github.com/borgbackup/borg/issues/747#issuecomment-1076160401"""
+        self.cmd('init', '--encryption=repokey', self.repository_location)
+        with Repository(self.repository_path) as repository:
+            key = msgpack.unpackb(a2b_base64(repository.load_key()))
+        assert key[b'algorithm'] == b'argon2 aes256-ctr hmac-sha256'
+
+    def test_init_with_explicit_key_algorithm(self):
+        """https://github.com/borgbackup/borg/issues/747#issuecomment-1076160401"""
+        self.cmd('init', '--encryption=repokey', '--key-algorithm=pbkdf2', self.repository_location)
+        with Repository(self.repository_path) as repository:
+            key = msgpack.unpackb(a2b_base64(repository.load_key()))
+        assert key[b'algorithm'] == b'sha256'
+
+    def verify_change_passphrase_does_not_change_algorithm(self, given_algorithm, expected_algorithm):
+        self.cmd('init', '--encryption=repokey', '--key-algorithm', given_algorithm, self.repository_location)
+        os.environ['BORG_NEW_PASSPHRASE'] = 'newpassphrase'
+
+        self.cmd('key', 'change-passphrase', self.repository_location)
+
+        with Repository(self.repository_path) as repository:
+            key = msgpack.unpackb(a2b_base64(repository.load_key()))
+            assert key[b'algorithm'] == expected_algorithm
+
+    def test_change_passphrase_does_not_change_algorithm_argon2(self):
+        self.verify_change_passphrase_does_not_change_algorithm('argon2', b'argon2 aes256-ctr hmac-sha256')
+
+    def test_change_passphrase_does_not_change_algorithm_pbkdf2(self):
+        self.verify_change_passphrase_does_not_change_algorithm('pbkdf2', b'sha256')
+
+    def verify_change_location_does_not_change_algorithm(self, given_algorithm, expected_algorithm):
+        self.cmd('init', '--encryption=keyfile', '--key-algorithm', given_algorithm, self.repository_location)
+
+        self.cmd('key', 'change-location', self.repository_location, 'repokey')
+
+        with Repository(self.repository_path) as repository:
+            key = msgpack.unpackb(a2b_base64(repository.load_key()))
+            assert key[b'algorithm'] == expected_algorithm
+
+    def test_change_location_does_not_change_algorithm_argon2(self):
+        self.verify_change_location_does_not_change_algorithm('argon2', b'argon2 aes256-ctr hmac-sha256')
+
+    def test_change_location_does_not_change_algorithm_pbkdf2(self):
+        self.verify_change_location_does_not_change_algorithm('pbkdf2', b'sha256')
+
 
 @unittest.skipUnless('binary' in BORG_EXES, 'no borg.exe available')
 class ArchiverTestCaseBinary(ArchiverTestCase):