Fix bn_assert_fits_in_bytes for big-endian

crypto/CMakeLists.txt

@@ -719,6 +719,7 @@ if(BUILD_TESTING)
     evp_extra/scrypt_test.cc
     fipsmodule/aes/aes_test.cc
     fipsmodule/bn/bn_test.cc
+    fipsmodule/bn/bn_assert_test.cc
     fipsmodule/cmac/cmac_test.cc
     fipsmodule/ec/ec_test.cc
     fipsmodule/ec/p256-nistz_test.cc

crypto/fipsmodule/bn/bn_assert_test.cc

@@ -0,0 +1,70 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0 OR ISC
+//
+
+#include <algorithm>
+#include <openssl/bn.h>
+#include <openssl/rand.h>
+#include "./internal.h"
+
+#include <gtest/gtest.h>
+
+TEST(BNAssertTest, assert_fits_in_bytes_large) {
+  bssl::UniquePtr<BIGNUM> x(BN_new());
+  uint8_t input[255];
+  OPENSSL_memset(input, 0, sizeof(input));
+  input[0] = 0xaa;
+  input[1] = 0x01;
+  input[254] = 0x01;
+  ASSERT_TRUE(BN_le2bn(input, sizeof(input), x.get()));
+  for (size_t i = 255; i < 260; i++) {
+    bn_assert_fits_in_bytes(x.get(), i);
+  }
+  for (size_t i = 247; i < 255; i++) {
+    EXPECT_DEBUG_DEATH(bn_assert_fits_in_bytes(x.get(), i), "");
+  }
+}
+
+TEST(BNAssertTest, assert_fits_in_bytes_small) {
+  bssl::UniquePtr<BIGNUM> x(BN_new());
+  uint8_t input[8];
+  OPENSSL_memset(input, 0, sizeof(input));
+  input[0] = 0xaa;
+  input[1] = 0xbb;
+  input[2] = 0xcc;
+  ASSERT_TRUE(BN_le2bn(input, sizeof(input), x.get()));
+
+  for (size_t i = 3; i < 10; i++) {
+    bn_assert_fits_in_bytes(x.get(), i);
+  }
+  for (size_t i = 0; i < 3; i++) {
+    EXPECT_DEBUG_DEATH(bn_assert_fits_in_bytes(x.get(), i), "");
+  }
+}
+
+TEST(BNAssertTest, assert_fits_in_bytes_zero) {
+  bssl::UniquePtr<BIGNUM> x(BN_new());
+  uint8_t input[8];
+  OPENSSL_memset(input, 0, sizeof(input));
+  ASSERT_TRUE(BN_le2bn(input, sizeof(input), x.get()));
+
+  for (size_t i = 0; i < 10; i++) {
+    bn_assert_fits_in_bytes(x.get(), i);
+  }
+}
+
+TEST(BNAssertTest, assert_fits_in_bytes_boundary) {
+  bssl::UniquePtr<BIGNUM> x(BN_new());
+  uint8_t input[8];
+  OPENSSL_memset(input, 0, sizeof(input));
+  for (size_t i = 0; i < sizeof(input); i++) {
+    input[i] = i * (i + 1) & 0xff;
+  }
+  ASSERT_TRUE(BN_le2bn(input, sizeof(input), x.get()));
+  for (size_t i = 8; i < 18; i++) {
+    bn_assert_fits_in_bytes(x.get(), i);
+  }
+  for (size_t i = 0; i < 8; i++) {
+    EXPECT_DEBUG_DEATH(bn_assert_fits_in_bytes(x.get(), i), "");
+  }
+}

crypto/fipsmodule/bn/bytes.c

@@ -205,14 +205,31 @@ static int fits_in_bytes(const BN_ULONG *words, size_t num_words,
   return mask == 0;
 }
 
+// Asserts that the BIGNUM can be represented within |num| bytes.
+// The logic is consistent with `fits_in_bytes` but assertions will fail when false.
 void bn_assert_fits_in_bytes(const BIGNUM *bn, size_t num) {
   const uint8_t *bytes = (const uint8_t *)bn->d;
   size_t tot_bytes = bn->width * sizeof(BN_ULONG);
   if (tot_bytes > num) {
     CONSTTIME_DECLASSIFY(bytes + num, tot_bytes - num);
+#ifdef OPENSSL_BIG_ENDIAN
+    for (int i = num / BN_BYTES; i < bn->width; i++) {
+      BN_ULONG word = bn->d[i];
+      for (size_t j = 0; j < BN_BYTES; j++) {
+        if ((i * BN_BYTES) + j < num) {
+          // For the first word we don't need to check any bytes shorter than len
+          continue;
+        } else {
+          uint8_t byte = (word >> (j * 8)) & 0xff;
+          assert(byte == 0);
+        }
+      }
+    }
+#else
     for (size_t i = num; i < tot_bytes; i++) {
       assert(bytes[i] == 0);
     }
+#endif
     (void)bytes;
   }
 }

crypto/fipsmodule/bn/internal.h

@@ -773,7 +773,7 @@ void bn_words_to_big_endian(uint8_t *out, size_t out_len, const BN_ULONG *in,
 // is in little-endian word order with |out[0]| being the least-significant word.
 // |out_len| must be large enough to represent any |in_len|-byte value. That is,
 // |out_len| must be at least |BN_BYTES * in_len|.
-void bn_little_endian_to_words(BN_ULONG *out, size_t out_len, const uint8_t *in, const size_t in_len);
+OPENSSL_EXPORT void bn_little_endian_to_words(BN_ULONG *out, size_t out_len, const uint8_t *in, const size_t in_len);
 
 // bn_words_to_little_endian represents |in_len| words from |in| (in little-endian
 // word order) as a little-endian, unsigned integer in |out_len| bytes. It