Skip to content

Commit

Permalink
Merge pull request #1034 from ThomasWaldmann/crypto-aead
Browse files Browse the repository at this point in the history
new crypto code, blackbox, aead internally
  • Loading branch information
enkore authored Jul 29, 2017
2 parents 8d89ee9 + dc4abff commit 7d02c7e
Show file tree
Hide file tree
Showing 13 changed files with 1,039 additions and 297 deletions.
3 changes: 2 additions & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@

compress_source = 'src/borg/compress.pyx'
crypto_ll_source = 'src/borg/crypto/low_level.pyx'
crypto_helpers = 'src/borg/crypto/_crypto_helpers.c'
chunker_source = 'src/borg/chunker.pyx'
hashindex_source = 'src/borg/hashindex.pyx'
item_source = 'src/borg/item.pyx'
Expand Down Expand Up @@ -730,7 +731,7 @@ def run(self):
if not on_rtd:
ext_modules += [
Extension('borg.compress', [compress_source], libraries=['lz4'], include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
Extension('borg.crypto.low_level', [crypto_ll_source], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
Extension('borg.crypto.low_level', [crypto_ll_source, crypto_helpers], libraries=crypto_libraries, include_dirs=include_dirs, library_dirs=library_dirs, define_macros=define_macros),
Extension('borg.hashindex', [hashindex_source]),
Extension('borg.item', [item_source]),
Extension('borg.chunker', [chunker_source]),
Expand Down
13 changes: 7 additions & 6 deletions src/borg/archive.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
from .crypto.key import key_factory
from .compress import Compressor, CompressionSpec
from .constants import * # NOQA
from .crypto.low_level import IntegrityError as IntegrityErrorBase
from .hashindex import ChunkIndex, ChunkIndexEntry, CacheSynchronizer
from .helpers import Manifest
from .helpers import hardlinkable
Expand Down Expand Up @@ -1148,7 +1149,7 @@ def check(self, repository, repair=False, archive=None, first=0, last=0, sort_by
else:
try:
self.manifest, _ = Manifest.load(repository, (Manifest.Operation.CHECK,), key=self.key)
except IntegrityError as exc:
except IntegrityErrorBase as exc:
logger.error('Repository manifest is corrupted: %s', exc)
self.error_found = True
del self.chunks[Manifest.MANIFEST_ID]
Expand Down Expand Up @@ -1211,11 +1212,11 @@ def verify_data(self):
chunk_id = chunk_ids_revd.pop(-1) # better efficiency
try:
encrypted_data = next(chunk_data_iter)
except (Repository.ObjectNotFound, IntegrityError) as err:
except (Repository.ObjectNotFound, IntegrityErrorBase) as err:
self.error_found = True
errors += 1
logger.error('chunk %s: %s', bin_to_hex(chunk_id), err)
if isinstance(err, IntegrityError):
if isinstance(err, IntegrityErrorBase):
defect_chunks.append(chunk_id)
# as the exception killed our generator, make a new one for remaining chunks:
if chunk_ids_revd:
Expand All @@ -1225,7 +1226,7 @@ def verify_data(self):
_chunk_id = None if chunk_id == Manifest.MANIFEST_ID else chunk_id
try:
self.key.decrypt(_chunk_id, encrypted_data)
except IntegrityError as integrity_error:
except IntegrityErrorBase as integrity_error:
self.error_found = True
errors += 1
logger.error('chunk %s, integrity error: %s', bin_to_hex(chunk_id), integrity_error)
Expand Down Expand Up @@ -1254,7 +1255,7 @@ def verify_data(self):
encrypted_data = self.repository.get(defect_chunk)
_chunk_id = None if defect_chunk == Manifest.MANIFEST_ID else defect_chunk
self.key.decrypt(_chunk_id, encrypted_data)
except IntegrityError:
except IntegrityErrorBase:
# failed twice -> get rid of this chunk
del self.chunks[defect_chunk]
self.repository.delete(defect_chunk)
Expand Down Expand Up @@ -1295,7 +1296,7 @@ def valid_archive(obj):
cdata = self.repository.get(chunk_id)
try:
data = self.key.decrypt(chunk_id, cdata)
except IntegrityError as exc:
except IntegrityErrorBase as exc:
logger.error('Skipping corrupted chunk: %s', exc)
self.error_found = True
continue
Expand Down
35 changes: 35 additions & 0 deletions src/borg/crypto/_crypto_helpers.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
/* some helpers, so our code also works with OpenSSL 1.0.x */

#include <string.h>
#include <openssl/opensslv.h>
#include <openssl/hmac.h>

#if OPENSSL_VERSION_NUMBER < 0x10100000L

HMAC_CTX *HMAC_CTX_new(void)
{
HMAC_CTX *ctx = OPENSSL_malloc(sizeof(*ctx));
if (ctx != NULL) {
memset(ctx, 0, sizeof *ctx);
HMAC_CTX_cleanup(ctx);
}
return ctx;
}

void HMAC_CTX_free(HMAC_CTX *ctx)
{
if (ctx != NULL) {
HMAC_CTX_cleanup(ctx);
OPENSSL_free(ctx);
}
}

const EVP_CIPHER *EVP_aes_256_ocb(void){ /* dummy, so that code compiles */
return NULL;
}

const EVP_CIPHER *EVP_chacha20_poly1305(void){ /* dummy, so that code compiles */
return NULL;
}

#endif
15 changes: 15 additions & 0 deletions src/borg/crypto/_crypto_helpers.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
/* some helpers, so our code also works with OpenSSL 1.0.x */

#include <openssl/opensslv.h>
#include <openssl/hmac.h>
#include <openssl/evp.h>

#if OPENSSL_VERSION_NUMBER < 0x10100000L

HMAC_CTX *HMAC_CTX_new(void);
void HMAC_CTX_free(HMAC_CTX *ctx);

const EVP_CIPHER *EVP_aes_256_ocb(void); /* dummy, so that code compiles */
const EVP_CIPHER *EVP_chacha20_poly1305(void); /* dummy, so that code compiles */

#endif
88 changes: 37 additions & 51 deletions src/borg/crypto/key.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@

import msgpack

from borg.logger import create_logger
from ..logger import create_logger

logger = create_logger()

Expand All @@ -25,10 +25,10 @@
from ..helpers import bin_to_hex
from ..item import Key, EncryptedKey
from ..platform import SaveFile
from .nonces import NonceManager
from .low_level import AES, bytes_to_long, bytes_to_int, num_aes_blocks, hmac_sha256, blake2b_256, hkdf_hmac_sha512

PREFIX = b'\0' * 8
from .nonces import NonceManager
from .low_level import AES, bytes_to_long, long_to_bytes, bytes_to_int, num_cipher_blocks, hmac_sha256, blake2b_256, hkdf_hmac_sha512
from .low_level import AES256_CTR_HMAC_SHA256, AES256_CTR_BLAKE2b


class PassphraseWrong(Error):
Expand Down Expand Up @@ -352,48 +352,31 @@ class AESKeyBase(KeyBase):

PAYLOAD_OVERHEAD = 1 + 32 + 8 # TYPE + HMAC + NONCE

MAC = hmac_sha256
CIPHERSUITE = AES256_CTR_HMAC_SHA256

logically_encrypted = True

def encrypt(self, chunk):
data = self.compressor.compress(chunk)
self.nonce_manager.ensure_reservation(num_aes_blocks(len(data)))
self.enc_cipher.reset()
data = b''.join((self.enc_cipher.iv[8:], self.enc_cipher.encrypt(data)))
assert (self.MAC is blake2b_256 and len(self.enc_hmac_key) == 128 or
self.MAC is hmac_sha256 and len(self.enc_hmac_key) == 32)
hmac = self.MAC(self.enc_hmac_key, data)
return b''.join((self.TYPE_STR, hmac, data))
next_iv = self.nonce_manager.ensure_reservation(self.cipher.next_iv(),
self.cipher.block_count(len(data)))
return self.cipher.encrypt(data, header=self.TYPE_STR, iv=next_iv)

def decrypt(self, id, data, decompress=True):
if not (data[0] == self.TYPE or
data[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
id_str = bin_to_hex(id) if id is not None else '(unknown)'
raise IntegrityError('Chunk %s: Invalid encryption envelope' % id_str)
data_view = memoryview(data)
hmac_given = data_view[1:33]
assert (self.MAC is blake2b_256 and len(self.enc_hmac_key) == 128 or
self.MAC is hmac_sha256 and len(self.enc_hmac_key) == 32)
hmac_computed = memoryview(self.MAC(self.enc_hmac_key, data_view[33:]))
if not compare_digest(hmac_computed, hmac_given):
id_str = bin_to_hex(id) if id is not None else '(unknown)'
raise IntegrityError('Chunk %s: Encryption envelope checksum mismatch' % id_str)
self.dec_cipher.reset(iv=PREFIX + data[33:41])
payload = self.dec_cipher.decrypt(data_view[41:])
try:
payload = self.cipher.decrypt(data)
except IntegrityError as e:
raise IntegrityError("Chunk %s: Could not decrypt [%s]" % (bin_to_hex(id), str(e)))
if not decompress:
return payload
data = self.decompress(payload)
self.assert_id(id, data)
return data

def extract_nonce(self, payload):
if not (payload[0] == self.TYPE or
payload[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
raise IntegrityError('Manifest: Invalid encryption envelope')
nonce = bytes_to_long(payload[33:41])
return nonce

def init_from_random_data(self, data=None):
if data is None:
data = os.urandom(100)
Expand All @@ -405,10 +388,21 @@ def init_from_random_data(self, data=None):
if self.chunk_seed & 0x80000000:
self.chunk_seed = self.chunk_seed - 0xffffffff - 1

def init_ciphers(self, manifest_nonce=0):
self.enc_cipher = AES(is_encrypt=True, key=self.enc_key, iv=manifest_nonce.to_bytes(16, byteorder='big'))
self.nonce_manager = NonceManager(self.repository, self.enc_cipher, manifest_nonce)
self.dec_cipher = AES(is_encrypt=False, key=self.enc_key)
def init_ciphers(self, manifest_data=None):
self.cipher = self.CIPHERSUITE(mac_key=self.enc_hmac_key, enc_key=self.enc_key, header_len=1, aad_offset=1)
if manifest_data is None:
nonce = 0
else:
if not (manifest_data[0] == self.TYPE or
manifest_data[0] == PassphraseKey.TYPE and isinstance(self, RepoKey)):
raise IntegrityError('Manifest: Invalid encryption envelope')
# manifest_blocks is a safe upper bound on the amount of cipher blocks needed
# to encrypt the manifest. depending on the ciphersuite and overhead, it might
# be a bit too high, but that does not matter.
manifest_blocks = num_cipher_blocks(len(manifest_data))
nonce = self.cipher.extract_iv(manifest_data) + manifest_blocks
self.cipher.set_iv(nonce)
self.nonce_manager = NonceManager(self.repository, nonce)


class Passphrase(str):
Expand Down Expand Up @@ -528,8 +522,7 @@ def detect(cls, repository, manifest_data):
key.init(repository, passphrase)
try:
key.decrypt(None, manifest_data)
num_blocks = num_aes_blocks(len(manifest_data) - 41)
key.init_ciphers(key.extract_nonce(manifest_data) + num_blocks)
key.init_ciphers(manifest_data)
key._passphrase = passphrase
return key
except IntegrityError:
Expand Down Expand Up @@ -568,8 +561,7 @@ def detect(cls, repository, manifest_data):
else:
if not key.load(target, passphrase):
raise PassphraseWrong
num_blocks = num_aes_blocks(len(manifest_data) - 41)
key.init_ciphers(key.extract_nonce(manifest_data) + num_blocks)
key.init_ciphers(manifest_data)
key._passphrase = passphrase
return key

Expand Down Expand Up @@ -604,7 +596,7 @@ def decrypt_key_file(self, data, passphrase):
assert enc_key.version == 1
assert enc_key.algorithm == 'sha256'
key = passphrase.kdf(enc_key.salt, enc_key.iterations, 32)
data = AES(is_encrypt=False, key=key).decrypt(enc_key.data)
data = AES(key, b'\0'*16).decrypt(enc_key.data)
if hmac_sha256(key, data) == enc_key.hash:
return data

Expand All @@ -613,7 +605,7 @@ def encrypt_key_file(self, data, passphrase):
iterations = PBKDF2_ITERATIONS
key = passphrase.kdf(salt, iterations, 32)
hash = hmac_sha256(key, data)
cdata = AES(is_encrypt=True, key=key).encrypt(data)
cdata = AES(key, b'\0'*16).encrypt(data)
enc_key = EncryptedKey(
version=1,
salt=salt,
Expand Down Expand Up @@ -772,7 +764,7 @@ class Blake2KeyfileKey(ID_BLAKE2b_256, KeyfileKey):
STORAGE = KeyBlobStorage.KEYFILE

FILE_ID = 'BORG_KEY'
MAC = blake2b_256
CIPHERSUITE = AES256_CTR_BLAKE2b


class Blake2RepoKey(ID_BLAKE2b_256, RepoKey):
Expand All @@ -781,7 +773,7 @@ class Blake2RepoKey(ID_BLAKE2b_256, RepoKey):
ARG_NAME = 'repokey-blake2'
STORAGE = KeyBlobStorage.REPO

MAC = blake2b_256
CIPHERSUITE = AES256_CTR_BLAKE2b


class AuthenticatedKeyBase(RepoKey):
Expand All @@ -799,24 +791,18 @@ def save(self, target, passphrase):
super().save(target, passphrase)
self.logically_encrypted = False

def extract_nonce(self, payload):
# This is called during set-up of the AES ciphers we're not actually using for this
# key. Therefore the return value of this method doesn't matter; it's just around
# to not have it crash should key identification be run against a very small chunk
# by "borg check" when the manifest is lost. (The manifest is always large enough
# to have the original method read some garbage from bytes 33-41). (Also, the return
# value must be larger than the 41 byte bloat of the original format).
if payload[0] != self.TYPE:
def init_ciphers(self, manifest_data=None):
if manifest_data is not None and manifest_data[0] != self.TYPE:
raise IntegrityError('Manifest: Invalid encryption envelope')
return 42

def encrypt(self, chunk):
data = self.compressor.compress(chunk)
return b''.join([self.TYPE_STR, data])

def decrypt(self, id, data, decompress=True):
if data[0] != self.TYPE:
raise IntegrityError('Chunk %s: Invalid envelope' % bin_to_hex(id))
id_str = bin_to_hex(id) if id is not None else '(unknown)'
raise IntegrityError('Chunk %s: Invalid envelope' % id_str)
payload = memoryview(data)[1:]
if not decompress:
return payload
Expand Down
Loading

0 comments on commit 7d02c7e

Please sign in to comment.