diff --git a/docs/installation.rst b/docs/installation.rst index 5b5d3dfb22..92f1a15ce6 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -164,7 +164,6 @@ following dependencies first: - liblz4_ >= 1.7.0 (r129) - libzstd_ >= 1.3.0 - - libb2_ >= 0.98.1 (older do not have pkg-config support) * pkg-config (cli tool) and pkgconfig python package (borg uses these to discover header and library location - if it can't import pkgconfig and is not pointed to header/library locations via env vars [see setup.py], @@ -198,7 +197,7 @@ Install the dependencies with development headers:: sudo apt-get install python3 python3-dev python3-pip python3-virtualenv \ libacl1-dev libacl1 \ - libssl-dev libb2-dev \ + libssl-dev \ liblz4-dev libzstd-dev \ build-essential \ pkg-config python3-pkgconfig @@ -216,7 +215,7 @@ Install the dependencies with development headers:: sudo dnf install python3 python3-devel python3-pip python3-virtualenv \ libacl-devel libacl \ - openssl-devel libb2-devel \ + openssl-devel \ lz4-devel libzstd-devel \ pkgconf python3-pkgconfig sudo dnf install gcc gcc-c++ redhat-rpm-config diff --git a/docs/internals/security.rst b/docs/internals/security.rst index d0c5bd1e35..3f79e66bd7 100644 --- a/docs/internals/security.rst +++ b/docs/internals/security.rst @@ -265,15 +265,13 @@ on widely used libraries providing them: We think this is not an additional risk, since we don't ever use OpenSSL's networking, TLS or X.509 code, but only their primitives implemented in libcrypto. -- SHA-256 and SHA-512 from Python's hashlib_ standard library module are used. +- SHA-256, SHA-512 and BLAKE2b from Python's hashlib_ standard library module are used. Borg requires a Python built with OpenSSL support (due to PBKDF2), therefore these functions are delegated to OpenSSL by Python. - HMAC, PBKDF2 and a constant-time comparison from Python's hmac_ standard library module is used. While the HMAC implementation is written in Python, the PBKDF2 implementation is provided by OpenSSL. The constant-time comparison (``compare_digest``) is written in C and part of Python. -- BLAKE2b is either provided by the system's libb2, an official implementation, - or a bundled copy of the BLAKE2 reference implementation (written in C). Implemented cryptographic constructions are: diff --git a/setup.py b/setup.py index 9da06cb1b2..a9ee6bdcf4 100644 --- a/setup.py +++ b/setup.py @@ -47,10 +47,6 @@ # needed: openssl >=1.0.2 or >=1.1.0 (or compatible) system_prefix_openssl = os.environ.get('BORG_OPENSSL_PREFIX') -# needed: blake2 (>= 0.98.1) -prefer_system_libb2 = not bool(os.environ.get('BORG_USE_BUNDLED_B2')) -system_prefix_libb2 = os.environ.get('BORG_LIBB2_PREFIX') - # needed: lz4 (>= 1.7.0 / r129) prefer_system_liblz4 = not bool(os.environ.get('BORG_USE_BUNDLED_LZ4')) system_prefix_liblz4 = os.environ.get('BORG_LIBLZ4_PREFIX') @@ -180,7 +176,6 @@ def members_appended(*ds): crypto_ext_kwargs = members_appended( dict(sources=[crypto_ll_source, crypto_helpers]), setup_crypto.crypto_ext_kwargs(pc, system_prefix_openssl), - setup_crypto.b2_ext_kwargs(pc, prefer_system_libb2, system_prefix_libb2), ) compress_ext_kwargs = members_appended( diff --git a/setup_crypto.py b/setup_crypto.py index ab316b1065..313bedbc7b 100644 --- a/setup_crypto.py +++ b/setup_crypto.py @@ -30,36 +30,3 @@ def crypto_ext_kwargs(pc, system_prefix): return pc.parse('libcrypto') raise Exception('Could not find OpenSSL lib/headers, please set BORG_OPENSSL_PREFIX') - - -# b2 files, structure as seen in BLAKE2 (reference implementation) project repository: - -# path relative (to this file) to the bundled library source code files -b2_bundled_path = 'src/borg/algorithms/blake2' - -b2_sources = [ - 'ref/blake2b-ref.c', -] - -b2_includes = [ - 'ref', -] - - -def b2_ext_kwargs(pc, prefer_system, system_prefix): - if prefer_system: - if system_prefix: - print('Detected and preferring libb2 [via BORG_LIBB2_PREFIX]') - return dict(include_dirs=[os.path.join(system_prefix, 'include')], - library_dirs=[os.path.join(system_prefix, 'lib')], - libraries=['b2']) - - if pc and pc.installed('libb2', '>= 0.98.1'): - print('Detected and preferring libb2 [via pkg-config]') - return pc.parse('libb2') - - print('Using bundled BLAKE2') - sources = multi_join(b2_sources, b2_bundled_path) - include_dirs = multi_join(b2_includes, b2_bundled_path) - define_macros = [('BORG_USE_BUNDLED_B2', 'YES')] - return dict(sources=sources, include_dirs=include_dirs, define_macros=define_macros) diff --git a/src/borg/algorithms/__init__.py b/src/borg/algorithms/__init__.py index cfa46c5196..32a0cfdbfe 100644 --- a/src/borg/algorithms/__init__.py +++ b/src/borg/algorithms/__init__.py @@ -5,7 +5,9 @@ This package is intended for hash and checksum functions. Ideally these would be sourced from existing libraries, -but are frequently not available yet (blake2), are -available but in poor form (crc32) or don't really -make sense as a library (xxHash). +but: + +- are frequently not available yet (lz4, zstd), +- are available but in poor form (crc32), or +- don't really make sense as a library (xxHash). """ diff --git a/src/borg/algorithms/blake2-libselect.h b/src/borg/algorithms/blake2-libselect.h deleted file mode 100644 index e0789e218e..0000000000 --- a/src/borg/algorithms/blake2-libselect.h +++ /dev/null @@ -1,5 +0,0 @@ -#ifdef BORG_USE_BUNDLED_B2 -#include "blake2/ref/blake2.h" -#else -#include -#endif diff --git a/src/borg/algorithms/blake2/ref/blake2-impl.h b/src/borg/algorithms/blake2/ref/blake2-impl.h deleted file mode 100644 index 5dff7fc7a3..0000000000 --- a/src/borg/algorithms/blake2/ref/blake2-impl.h +++ /dev/null @@ -1,160 +0,0 @@ -/* - BLAKE2 reference source code package - reference C implementations - - Copyright 2012, Samuel Neves . You may use this under the - terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at - your option. The terms of these licenses can be found at: - - - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 - - OpenSSL license : https://www.openssl.org/source/license.html - - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 - - More information about the BLAKE2 hash function can be found at - https://blake2.net. -*/ -#ifndef BLAKE2_IMPL_H -#define BLAKE2_IMPL_H - -#include -#include - -#if !defined(__cplusplus) && (!defined(__STDC_VERSION__) || __STDC_VERSION__ < 199901L) - #if defined(_MSC_VER) - #define BLAKE2_INLINE __inline - #elif defined(__GNUC__) - #define BLAKE2_INLINE __inline__ - #else - #define BLAKE2_INLINE - #endif -#else - #define BLAKE2_INLINE inline -#endif - -static BLAKE2_INLINE uint32_t load32( const void *src ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - uint32_t w; - memcpy(&w, src, sizeof w); - return w; -#else - const uint8_t *p = ( const uint8_t * )src; - return (( uint32_t )( p[0] ) << 0) | - (( uint32_t )( p[1] ) << 8) | - (( uint32_t )( p[2] ) << 16) | - (( uint32_t )( p[3] ) << 24) ; -#endif -} - -static BLAKE2_INLINE uint64_t load64( const void *src ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - uint64_t w; - memcpy(&w, src, sizeof w); - return w; -#else - const uint8_t *p = ( const uint8_t * )src; - return (( uint64_t )( p[0] ) << 0) | - (( uint64_t )( p[1] ) << 8) | - (( uint64_t )( p[2] ) << 16) | - (( uint64_t )( p[3] ) << 24) | - (( uint64_t )( p[4] ) << 32) | - (( uint64_t )( p[5] ) << 40) | - (( uint64_t )( p[6] ) << 48) | - (( uint64_t )( p[7] ) << 56) ; -#endif -} - -static BLAKE2_INLINE uint16_t load16( const void *src ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - uint16_t w; - memcpy(&w, src, sizeof w); - return w; -#else - const uint8_t *p = ( const uint8_t * )src; - return (( uint16_t )( p[0] ) << 0) | - (( uint16_t )( p[1] ) << 8) ; -#endif -} - -static BLAKE2_INLINE void store16( void *dst, uint16_t w ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - memcpy(dst, &w, sizeof w); -#else - uint8_t *p = ( uint8_t * )dst; - *p++ = ( uint8_t )w; w >>= 8; - *p++ = ( uint8_t )w; -#endif -} - -static BLAKE2_INLINE void store32( void *dst, uint32_t w ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - memcpy(dst, &w, sizeof w); -#else - uint8_t *p = ( uint8_t * )dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); -#endif -} - -static BLAKE2_INLINE void store64( void *dst, uint64_t w ) -{ -#if defined(NATIVE_LITTLE_ENDIAN) - memcpy(dst, &w, sizeof w); -#else - uint8_t *p = ( uint8_t * )dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); - p[4] = (uint8_t)(w >> 32); - p[5] = (uint8_t)(w >> 40); - p[6] = (uint8_t)(w >> 48); - p[7] = (uint8_t)(w >> 56); -#endif -} - -static BLAKE2_INLINE uint64_t load48( const void *src ) -{ - const uint8_t *p = ( const uint8_t * )src; - return (( uint64_t )( p[0] ) << 0) | - (( uint64_t )( p[1] ) << 8) | - (( uint64_t )( p[2] ) << 16) | - (( uint64_t )( p[3] ) << 24) | - (( uint64_t )( p[4] ) << 32) | - (( uint64_t )( p[5] ) << 40) ; -} - -static BLAKE2_INLINE void store48( void *dst, uint64_t w ) -{ - uint8_t *p = ( uint8_t * )dst; - p[0] = (uint8_t)(w >> 0); - p[1] = (uint8_t)(w >> 8); - p[2] = (uint8_t)(w >> 16); - p[3] = (uint8_t)(w >> 24); - p[4] = (uint8_t)(w >> 32); - p[5] = (uint8_t)(w >> 40); -} - -static BLAKE2_INLINE uint32_t rotr32( const uint32_t w, const unsigned c ) -{ - return ( w >> c ) | ( w << ( 32 - c ) ); -} - -static BLAKE2_INLINE uint64_t rotr64( const uint64_t w, const unsigned c ) -{ - return ( w >> c ) | ( w << ( 64 - c ) ); -} - -/* prevents compiler optimizing out memset() */ -static BLAKE2_INLINE void secure_zero_memory(void *v, size_t n) -{ - static void *(*const volatile memset_v)(void *, int, size_t) = &memset; - memset_v(v, 0, n); -} - -#endif diff --git a/src/borg/algorithms/blake2/ref/blake2.h b/src/borg/algorithms/blake2/ref/blake2.h deleted file mode 100644 index ad62f260e7..0000000000 --- a/src/borg/algorithms/blake2/ref/blake2.h +++ /dev/null @@ -1,195 +0,0 @@ -/* - BLAKE2 reference source code package - reference C implementations - - Copyright 2012, Samuel Neves . You may use this under the - terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at - your option. The terms of these licenses can be found at: - - - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 - - OpenSSL license : https://www.openssl.org/source/license.html - - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 - - More information about the BLAKE2 hash function can be found at - https://blake2.net. -*/ -#ifndef BLAKE2_H -#define BLAKE2_H - -#include -#include - -#if defined(_MSC_VER) -#define BLAKE2_PACKED(x) __pragma(pack(push, 1)) x __pragma(pack(pop)) -#else -#define BLAKE2_PACKED(x) x __attribute__((packed)) -#endif - -#if defined(__cplusplus) -extern "C" { -#endif - - enum blake2s_constant - { - BLAKE2S_BLOCKBYTES = 64, - BLAKE2S_OUTBYTES = 32, - BLAKE2S_KEYBYTES = 32, - BLAKE2S_SALTBYTES = 8, - BLAKE2S_PERSONALBYTES = 8 - }; - - enum blake2b_constant - { - BLAKE2B_BLOCKBYTES = 128, - BLAKE2B_OUTBYTES = 64, - BLAKE2B_KEYBYTES = 64, - BLAKE2B_SALTBYTES = 16, - BLAKE2B_PERSONALBYTES = 16 - }; - - typedef struct blake2s_state__ - { - uint32_t h[8]; - uint32_t t[2]; - uint32_t f[2]; - uint8_t buf[BLAKE2S_BLOCKBYTES]; - size_t buflen; - size_t outlen; - uint8_t last_node; - } blake2s_state; - - typedef struct blake2b_state__ - { - uint64_t h[8]; - uint64_t t[2]; - uint64_t f[2]; - uint8_t buf[BLAKE2B_BLOCKBYTES]; - size_t buflen; - size_t outlen; - uint8_t last_node; - } blake2b_state; - - typedef struct blake2sp_state__ - { - blake2s_state S[8][1]; - blake2s_state R[1]; - uint8_t buf[8 * BLAKE2S_BLOCKBYTES]; - size_t buflen; - size_t outlen; - } blake2sp_state; - - typedef struct blake2bp_state__ - { - blake2b_state S[4][1]; - blake2b_state R[1]; - uint8_t buf[4 * BLAKE2B_BLOCKBYTES]; - size_t buflen; - size_t outlen; - } blake2bp_state; - - - BLAKE2_PACKED(struct blake2s_param__ - { - uint8_t digest_length; /* 1 */ - uint8_t key_length; /* 2 */ - uint8_t fanout; /* 3 */ - uint8_t depth; /* 4 */ - uint32_t leaf_length; /* 8 */ - uint32_t node_offset; /* 12 */ - uint16_t xof_length; /* 14 */ - uint8_t node_depth; /* 15 */ - uint8_t inner_length; /* 16 */ - /* uint8_t reserved[0]; */ - uint8_t salt[BLAKE2S_SALTBYTES]; /* 24 */ - uint8_t personal[BLAKE2S_PERSONALBYTES]; /* 32 */ - }); - - typedef struct blake2s_param__ blake2s_param; - - BLAKE2_PACKED(struct blake2b_param__ - { - uint8_t digest_length; /* 1 */ - uint8_t key_length; /* 2 */ - uint8_t fanout; /* 3 */ - uint8_t depth; /* 4 */ - uint32_t leaf_length; /* 8 */ - uint32_t node_offset; /* 12 */ - uint32_t xof_length; /* 16 */ - uint8_t node_depth; /* 17 */ - uint8_t inner_length; /* 18 */ - uint8_t reserved[14]; /* 32 */ - uint8_t salt[BLAKE2B_SALTBYTES]; /* 48 */ - uint8_t personal[BLAKE2B_PERSONALBYTES]; /* 64 */ - }); - - typedef struct blake2b_param__ blake2b_param; - - typedef struct blake2xs_state__ - { - blake2s_state S[1]; - blake2s_param P[1]; - } blake2xs_state; - - typedef struct blake2xb_state__ - { - blake2b_state S[1]; - blake2b_param P[1]; - } blake2xb_state; - - /* Padded structs result in a compile-time error */ - enum { - BLAKE2_DUMMY_1 = 1/(sizeof(blake2s_param) == BLAKE2S_OUTBYTES), - BLAKE2_DUMMY_2 = 1/(sizeof(blake2b_param) == BLAKE2B_OUTBYTES) - }; - - /* Streaming API */ - int blake2s_init( blake2s_state *S, size_t outlen ); - int blake2s_init_key( blake2s_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2s_init_param( blake2s_state *S, const blake2s_param *P ); - int blake2s_update( blake2s_state *S, const void *in, size_t inlen ); - int blake2s_final( blake2s_state *S, void *out, size_t outlen ); - - int blake2b_init( blake2b_state *S, size_t outlen ); - int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2b_init_param( blake2b_state *S, const blake2b_param *P ); - int blake2b_update( blake2b_state *S, const void *in, size_t inlen ); - int blake2b_final( blake2b_state *S, void *out, size_t outlen ); - - int blake2sp_init( blake2sp_state *S, size_t outlen ); - int blake2sp_init_key( blake2sp_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2sp_update( blake2sp_state *S, const void *in, size_t inlen ); - int blake2sp_final( blake2sp_state *S, void *out, size_t outlen ); - - int blake2bp_init( blake2bp_state *S, size_t outlen ); - int blake2bp_init_key( blake2bp_state *S, size_t outlen, const void *key, size_t keylen ); - int blake2bp_update( blake2bp_state *S, const void *in, size_t inlen ); - int blake2bp_final( blake2bp_state *S, void *out, size_t outlen ); - - /* Variable output length API */ - int blake2xs_init( blake2xs_state *S, const size_t outlen ); - int blake2xs_init_key( blake2xs_state *S, const size_t outlen, const void *key, size_t keylen ); - int blake2xs_update( blake2xs_state *S, const void *in, size_t inlen ); - int blake2xs_final(blake2xs_state *S, void *out, size_t outlen); - - int blake2xb_init( blake2xb_state *S, const size_t outlen ); - int blake2xb_init_key( blake2xb_state *S, const size_t outlen, const void *key, size_t keylen ); - int blake2xb_update( blake2xb_state *S, const void *in, size_t inlen ); - int blake2xb_final(blake2xb_state *S, void *out, size_t outlen); - - /* Simple API */ - int blake2s( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - - int blake2sp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - int blake2bp( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - - int blake2xs( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - int blake2xb( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - - /* This is simply an alias for blake2b */ - int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ); - -#if defined(__cplusplus) -} -#endif - -#endif diff --git a/src/borg/algorithms/blake2/ref/blake2b-ref.c b/src/borg/algorithms/blake2/ref/blake2b-ref.c deleted file mode 100644 index cd38b1ba00..0000000000 --- a/src/borg/algorithms/blake2/ref/blake2b-ref.c +++ /dev/null @@ -1,379 +0,0 @@ -/* - BLAKE2 reference source code package - reference C implementations - - Copyright 2012, Samuel Neves . You may use this under the - terms of the CC0, the OpenSSL Licence, or the Apache Public License 2.0, at - your option. The terms of these licenses can be found at: - - - CC0 1.0 Universal : http://creativecommons.org/publicdomain/zero/1.0 - - OpenSSL license : https://www.openssl.org/source/license.html - - Apache 2.0 : http://www.apache.org/licenses/LICENSE-2.0 - - More information about the BLAKE2 hash function can be found at - https://blake2.net. -*/ - -#include -#include -#include - -#include "blake2.h" -#include "blake2-impl.h" - -static const uint64_t blake2b_IV[8] = -{ - 0x6a09e667f3bcc908ULL, 0xbb67ae8584caa73bULL, - 0x3c6ef372fe94f82bULL, 0xa54ff53a5f1d36f1ULL, - 0x510e527fade682d1ULL, 0x9b05688c2b3e6c1fULL, - 0x1f83d9abfb41bd6bULL, 0x5be0cd19137e2179ULL -}; - -static const uint8_t blake2b_sigma[12][16] = -{ - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } , - { 11, 8, 12, 0, 5, 2, 15, 13, 10, 14, 3, 6, 7, 1, 9, 4 } , - { 7, 9, 3, 1, 13, 12, 11, 14, 2, 6, 5, 10, 4, 0, 15, 8 } , - { 9, 0, 5, 7, 2, 4, 10, 15, 14, 1, 11, 12, 6, 8, 3, 13 } , - { 2, 12, 6, 10, 0, 11, 8, 3, 4, 13, 7, 5, 15, 14, 1, 9 } , - { 12, 5, 1, 15, 14, 13, 4, 10, 0, 7, 6, 3, 9, 2, 8, 11 } , - { 13, 11, 7, 14, 12, 1, 3, 9, 5, 0, 15, 4, 8, 6, 2, 10 } , - { 6, 15, 14, 9, 11, 3, 0, 8, 12, 2, 13, 7, 1, 4, 10, 5 } , - { 10, 2, 8, 4, 7, 6, 1, 5, 15, 11, 9, 14, 3, 12, 13 , 0 } , - { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 } , - { 14, 10, 4, 8, 9, 15, 13, 6, 1, 12, 0, 2, 11, 7, 5, 3 } -}; - - -static void blake2b_set_lastnode( blake2b_state *S ) -{ - S->f[1] = (uint64_t)-1; -} - -/* Some helper functions, not necessarily useful */ -static int blake2b_is_lastblock( const blake2b_state *S ) -{ - return S->f[0] != 0; -} - -static void blake2b_set_lastblock( blake2b_state *S ) -{ - if( S->last_node ) blake2b_set_lastnode( S ); - - S->f[0] = (uint64_t)-1; -} - -static void blake2b_increment_counter( blake2b_state *S, const uint64_t inc ) -{ - S->t[0] += inc; - S->t[1] += ( S->t[0] < inc ); -} - -static void blake2b_init0( blake2b_state *S ) -{ - size_t i; - memset( S, 0, sizeof( blake2b_state ) ); - - for( i = 0; i < 8; ++i ) S->h[i] = blake2b_IV[i]; -} - -/* init xors IV with input parameter block */ -int blake2b_init_param( blake2b_state *S, const blake2b_param *P ) -{ - const uint8_t *p = ( const uint8_t * )( P ); - size_t i; - - blake2b_init0( S ); - - /* IV XOR ParamBlock */ - for( i = 0; i < 8; ++i ) - S->h[i] ^= load64( p + sizeof( S->h[i] ) * i ); - - S->outlen = P->digest_length; - return 0; -} - - - -int blake2b_init( blake2b_state *S, size_t outlen ) -{ - blake2b_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - P->digest_length = (uint8_t)outlen; - P->key_length = 0; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store32( &P->node_offset, 0 ); - store32( &P->xof_length, 0 ); - P->node_depth = 0; - P->inner_length = 0; - memset( P->reserved, 0, sizeof( P->reserved ) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - return blake2b_init_param( S, P ); -} - - -int blake2b_init_key( blake2b_state *S, size_t outlen, const void *key, size_t keylen ) -{ - blake2b_param P[1]; - - if ( ( !outlen ) || ( outlen > BLAKE2B_OUTBYTES ) ) return -1; - - if ( !key || !keylen || keylen > BLAKE2B_KEYBYTES ) return -1; - - P->digest_length = (uint8_t)outlen; - P->key_length = (uint8_t)keylen; - P->fanout = 1; - P->depth = 1; - store32( &P->leaf_length, 0 ); - store32( &P->node_offset, 0 ); - store32( &P->xof_length, 0 ); - P->node_depth = 0; - P->inner_length = 0; - memset( P->reserved, 0, sizeof( P->reserved ) ); - memset( P->salt, 0, sizeof( P->salt ) ); - memset( P->personal, 0, sizeof( P->personal ) ); - - if( blake2b_init_param( S, P ) < 0 ) return -1; - - { - uint8_t block[BLAKE2B_BLOCKBYTES]; - memset( block, 0, BLAKE2B_BLOCKBYTES ); - memcpy( block, key, keylen ); - blake2b_update( S, block, BLAKE2B_BLOCKBYTES ); - secure_zero_memory( block, BLAKE2B_BLOCKBYTES ); /* Burn the key from stack */ - } - return 0; -} - -#define G(r,i,a,b,c,d) \ - do { \ - a = a + b + m[blake2b_sigma[r][2*i+0]]; \ - d = rotr64(d ^ a, 32); \ - c = c + d; \ - b = rotr64(b ^ c, 24); \ - a = a + b + m[blake2b_sigma[r][2*i+1]]; \ - d = rotr64(d ^ a, 16); \ - c = c + d; \ - b = rotr64(b ^ c, 63); \ - } while(0) - -#define ROUND(r) \ - do { \ - G(r,0,v[ 0],v[ 4],v[ 8],v[12]); \ - G(r,1,v[ 1],v[ 5],v[ 9],v[13]); \ - G(r,2,v[ 2],v[ 6],v[10],v[14]); \ - G(r,3,v[ 3],v[ 7],v[11],v[15]); \ - G(r,4,v[ 0],v[ 5],v[10],v[15]); \ - G(r,5,v[ 1],v[ 6],v[11],v[12]); \ - G(r,6,v[ 2],v[ 7],v[ 8],v[13]); \ - G(r,7,v[ 3],v[ 4],v[ 9],v[14]); \ - } while(0) - -static void blake2b_compress( blake2b_state *S, const uint8_t block[BLAKE2B_BLOCKBYTES] ) -{ - uint64_t m[16]; - uint64_t v[16]; - size_t i; - - for( i = 0; i < 16; ++i ) { - m[i] = load64( block + i * sizeof( m[i] ) ); - } - - for( i = 0; i < 8; ++i ) { - v[i] = S->h[i]; - } - - v[ 8] = blake2b_IV[0]; - v[ 9] = blake2b_IV[1]; - v[10] = blake2b_IV[2]; - v[11] = blake2b_IV[3]; - v[12] = blake2b_IV[4] ^ S->t[0]; - v[13] = blake2b_IV[5] ^ S->t[1]; - v[14] = blake2b_IV[6] ^ S->f[0]; - v[15] = blake2b_IV[7] ^ S->f[1]; - - ROUND( 0 ); - ROUND( 1 ); - ROUND( 2 ); - ROUND( 3 ); - ROUND( 4 ); - ROUND( 5 ); - ROUND( 6 ); - ROUND( 7 ); - ROUND( 8 ); - ROUND( 9 ); - ROUND( 10 ); - ROUND( 11 ); - - for( i = 0; i < 8; ++i ) { - S->h[i] = S->h[i] ^ v[i] ^ v[i + 8]; - } -} - -#undef G -#undef ROUND - -int blake2b_update( blake2b_state *S, const void *pin, size_t inlen ) -{ - const unsigned char * in = (const unsigned char *)pin; - if( inlen > 0 ) - { - size_t left = S->buflen; - size_t fill = BLAKE2B_BLOCKBYTES - left; - if( inlen > fill ) - { - S->buflen = 0; - memcpy( S->buf + left, in, fill ); /* Fill buffer */ - blake2b_increment_counter( S, BLAKE2B_BLOCKBYTES ); - blake2b_compress( S, S->buf ); /* Compress */ - in += fill; inlen -= fill; - while(inlen > BLAKE2B_BLOCKBYTES) { - blake2b_increment_counter(S, BLAKE2B_BLOCKBYTES); - blake2b_compress( S, in ); - in += BLAKE2B_BLOCKBYTES; - inlen -= BLAKE2B_BLOCKBYTES; - } - } - memcpy( S->buf + S->buflen, in, inlen ); - S->buflen += inlen; - } - return 0; -} - -int blake2b_final( blake2b_state *S, void *out, size_t outlen ) -{ - uint8_t buffer[BLAKE2B_OUTBYTES] = {0}; - size_t i; - - if( out == NULL || outlen < S->outlen ) - return -1; - - if( blake2b_is_lastblock( S ) ) - return -1; - - blake2b_increment_counter( S, S->buflen ); - blake2b_set_lastblock( S ); - memset( S->buf + S->buflen, 0, BLAKE2B_BLOCKBYTES - S->buflen ); /* Padding */ - blake2b_compress( S, S->buf ); - - for( i = 0; i < 8; ++i ) /* Output full hash to temp buffer */ - store64( buffer + sizeof( S->h[i] ) * i, S->h[i] ); - - memcpy( out, buffer, S->outlen ); - secure_zero_memory(buffer, sizeof(buffer)); - return 0; -} - -/* inlen, at least, should be uint64_t. Others can be size_t. */ -int blake2b( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) -{ - blake2b_state S[1]; - - /* Verify parameters */ - if ( NULL == in && inlen > 0 ) return -1; - - if ( NULL == out ) return -1; - - if( NULL == key && keylen > 0 ) return -1; - - if( !outlen || outlen > BLAKE2B_OUTBYTES ) return -1; - - if( keylen > BLAKE2B_KEYBYTES ) return -1; - - if( keylen > 0 ) - { - if( blake2b_init_key( S, outlen, key, keylen ) < 0 ) return -1; - } - else - { - if( blake2b_init( S, outlen ) < 0 ) return -1; - } - - blake2b_update( S, ( const uint8_t * )in, inlen ); - blake2b_final( S, out, outlen ); - return 0; -} - -int blake2( void *out, size_t outlen, const void *in, size_t inlen, const void *key, size_t keylen ) { - return blake2b(out, outlen, in, inlen, key, keylen); -} - -#if defined(SUPERCOP) -int crypto_hash( unsigned char *out, unsigned char *in, unsigned long long inlen ) -{ - return blake2b( out, BLAKE2B_OUTBYTES, in, inlen, NULL, 0 ); -} -#endif - -#if defined(BLAKE2B_SELFTEST) -#include -#include "blake2-kat.h" -int main( void ) -{ - uint8_t key[BLAKE2B_KEYBYTES]; - uint8_t buf[BLAKE2_KAT_LENGTH]; - size_t i, step; - - for( i = 0; i < BLAKE2B_KEYBYTES; ++i ) - key[i] = ( uint8_t )i; - - for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) - buf[i] = ( uint8_t )i; - - /* Test simple API */ - for( i = 0; i < BLAKE2_KAT_LENGTH; ++i ) - { - uint8_t hash[BLAKE2B_OUTBYTES]; - blake2b( hash, BLAKE2B_OUTBYTES, buf, i, key, BLAKE2B_KEYBYTES ); - - if( 0 != memcmp( hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES ) ) - { - goto fail; - } - } - - /* Test streaming API */ - for(step = 1; step < BLAKE2B_BLOCKBYTES; ++step) { - for (i = 0; i < BLAKE2_KAT_LENGTH; ++i) { - uint8_t hash[BLAKE2B_OUTBYTES]; - blake2b_state S; - uint8_t * p = buf; - size_t mlen = i; - int err = 0; - - if( (err = blake2b_init_key(&S, BLAKE2B_OUTBYTES, key, BLAKE2B_KEYBYTES)) < 0 ) { - goto fail; - } - - while (mlen >= step) { - if ( (err = blake2b_update(&S, p, step)) < 0 ) { - goto fail; - } - mlen -= step; - p += step; - } - if ( (err = blake2b_update(&S, p, mlen)) < 0) { - goto fail; - } - if ( (err = blake2b_final(&S, hash, BLAKE2B_OUTBYTES)) < 0) { - goto fail; - } - - if (0 != memcmp(hash, blake2b_keyed_kat[i], BLAKE2B_OUTBYTES)) { - goto fail; - } - } - } - - puts( "ok" ); - return 0; -fail: - puts("error"); - return -1; -} -#endif diff --git a/src/borg/crypto/low_level.pyx b/src/borg/crypto/low_level.pyx index 9c09fe0794..1f23a126ab 100644 --- a/src/borg/crypto/low_level.pyx +++ b/src/borg/crypto/low_level.pyx @@ -47,14 +47,6 @@ API_VERSION = '1.2_01' cdef extern from "openssl/crypto.h": int CRYPTO_memcmp(const void *a, const void *b, size_t len) -cdef extern from "../algorithms/blake2-libselect.h": - ctypedef struct blake2b_state: - pass - - int blake2b_init(blake2b_state *S, size_t outlen) nogil - int blake2b_update(blake2b_state *S, const void *input, size_t inlen) nogil - int blake2b_final(blake2b_state *S, void *out, size_t outlen) nogil - cdef extern from "openssl/evp.h": ctypedef struct EVP_MD: @@ -439,22 +431,10 @@ cdef class AES256_CTR_BLAKE2b(AES256_CTR_BASE): cdef mac_compute(self, const unsigned char *data1, int data1_len, const unsigned char *data2, int data2_len, unsigned char *mac_buf): - cdef blake2b_state state - cdef int rc - rc = blake2b_init(&state, self.mac_len) - if rc == -1: - raise Exception('blake2b_init() failed') - with nogil: - rc = blake2b_update(&state, self.mac_key, 128) - if rc != -1: - rc = blake2b_update(&state, data1, data1_len) - if rc != -1: - rc = blake2b_update(&state, data2, data2_len) - if rc == -1: - raise Exception('blake2b_update() failed') - rc = blake2b_final(&state, mac_buf, self.mac_len) - if rc == -1: - raise Exception('blake2b_final() failed') + data = self.mac_key[:128] + data1[:data1_len] + data2[:data2_len] + mac = hashlib.blake2b(data, digest_size=self.mac_len).digest() + for i in range(self.mac_len): + mac_buf[i] = mac[i] cdef mac_verify(self, const unsigned char *data1, int data1_len, const unsigned char *data2, int data2_len, @@ -813,59 +793,12 @@ def hmac_sha256(key, data): return PyBytes_FromStringAndSize( &md[0], 32) -cdef blake2b_update_from_buffer(blake2b_state *state, obj): - cdef Py_buffer buf = ro_buffer(obj) - try: - with nogil: - rc = blake2b_update(state, buf.buf, buf.len) - if rc == -1: - raise Exception('blake2b_update() failed') - finally: - PyBuffer_Release(&buf) - - def blake2b_256(key, data): - cdef blake2b_state state - if blake2b_init(&state, 32) == -1: - raise Exception('blake2b_init() failed') - - cdef unsigned char md[32] - cdef unsigned char *key_ptr = key - - # This is secure, because BLAKE2 is not vulnerable to length-extension attacks (unlike SHA-1/2, MD-5 and others). - # See the BLAKE2 paper section 2.9 "Keyed hashing (MAC and PRF)" for details. - # A nice benefit is that this simpler prefix-MAC mode has less overhead than the more complex HMAC mode. - # We don't use the BLAKE2 parameter block (via blake2s_init_key) for this to - # avoid incompatibility with the limited API of OpenSSL. - rc = blake2b_update(&state, key_ptr, len(key)) - if rc == -1: - raise Exception('blake2b_update() failed') - blake2b_update_from_buffer(&state, data) - - rc = blake2b_final(&state, &md[0], 32) - if rc == -1: - raise Exception('blake2b_final() failed') - - return PyBytes_FromStringAndSize( &md[0], 32) + return hashlib.blake2b(key+data, digest_size=32).digest() def blake2b_128(data): - cdef blake2b_state state - cdef unsigned char md[16] - cdef unsigned char *data_ptr = data - - if blake2b_init(&state, 16) == -1: - raise Exception('blake2b_init() failed') - - rc = blake2b_update(&state, data_ptr, len(data)) - if rc == -1: - raise Exception('blake2b_update() failed') - - rc = blake2b_final(&state, &md[0], 16) - if rc == -1: - raise Exception('blake2b_final() failed') - - return PyBytes_FromStringAndSize( &md[0], 16) + return hashlib.blake2b(data, digest_size=16).digest() def hkdf_hmac_sha512(ikm, salt, info, output_length): diff --git a/src/borg/selftest.py b/src/borg/selftest.py index 80707c0cc2..49ad0fbf1c 100644 --- a/src/borg/selftest.py +++ b/src/borg/selftest.py @@ -30,7 +30,7 @@ ChunkerTestCase, ] -SELFTEST_COUNT = 37 +SELFTEST_COUNT = 38 class SelfTestResult(TestResult): diff --git a/src/borg/testsuite/crypto.py b/src/borg/testsuite/crypto.py index 407a4e9abc..bf15887e9d 100644 --- a/src/borg/testsuite/crypto.py +++ b/src/borg/testsuite/crypto.py @@ -1,7 +1,7 @@ from binascii import hexlify, unhexlify from ..crypto.low_level import AES256_CTR_HMAC_SHA256, AES256_OCB, CHACHA20_POLY1305, UNENCRYPTED, \ - IntegrityError, blake2b_256, hmac_sha256, openssl10 + IntegrityError, blake2b_128, blake2b_256, hmac_sha256, openssl10 from ..crypto.low_level import bytes_to_long, bytes_to_int, long_to_bytes from ..crypto.low_level import hkdf_hmac_sha512 @@ -219,6 +219,12 @@ def test_blake2b_256(self): data = memoryview(b'1234567890' * 100) assert blake2b_256(key, data) == unhexlify('97ede832378531dd0f4c668685d166e797da27b47d8cd441e885b60abd5e0cb2') + def test_blake2b_128(self): + # (see above) + assert blake2b_128(b'') == unhexlify('cae66941d9efbd404e4d88758ea67670') + assert blake2b_128(b'abc') == unhexlify('cf4ab791c62b8d2b2109c90275287816') + assert blake2b_128(b'abcd'*8) == unhexlify('0f759d9a32d3f99250c1781a8baa58b9') + # These test vectors come from https://www.kullo.net/blog/hkdf-sha-512-test-vectors/ # who claims to have verified these against independent Python and C++ implementations.