Fix bn_assert_fits_in_bytes for big-endian

crypto/CMakeLists.txt

@@ -719,6 +719,7 @@ if(BUILD_TESTING)
     evp_extra/scrypt_test.cc
     fipsmodule/aes/aes_test.cc
     fipsmodule/bn/bn_test.cc
+    fipsmodule/bn/bn_assert_test.cc
     fipsmodule/cmac/cmac_test.cc
     fipsmodule/ec/ec_test.cc
     fipsmodule/ec/p256-nistz_test.cc

crypto/fipsmodule/bn/bn_assert_test.cc

@@ -0,0 +1,70 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: Apache-2.0 OR ISC
+//
+
+#include <algorithm>
+#include <openssl/bn.h>
+#include <openssl/rand.h>
+#include "./internal.h"
+
+#include <gtest/gtest.h>
+
+TEST(BNAssertTest, assert_fits_in_bytes_large) {
+  bssl::UniquePtr<BIGNUM> x(BN_new());
+  uint8_t input[255];
+  OPENSSL_memset(input, 0, sizeof(input));
+  input[0] = 0xaa;
+  input[1] = 0x01;
+  input[254] = 0x01;
+  ASSERT_TRUE(BN_le2bn(input, sizeof(input), x.get()));
+  for (size_t i = 255; i < 260; i++) {
+    bn_assert_fits_in_bytes(x.get(), i);
+  }
+  for (size_t i = 247; i < 255; i++) {
+    EXPECT_DEATH(bn_assert_fits_in_bytes(x.get(), i), "");
+  }
+}
+
+TEST(BNAssertTest, assert_fits_in_bytes_small) {
+  bssl::UniquePtr<BIGNUM> x(BN_new());
+  uint8_t input[8];
+  OPENSSL_memset(input, 0, sizeof(input));
+  input[0] = 0xaa;
+  input[1] = 0xbb;
+  input[2] = 0xcc;
+  ASSERT_TRUE(BN_le2bn(input, sizeof(input), x.get()));
+
+  for (size_t i = 3; i < 10; i++) {
+    bn_assert_fits_in_bytes(x.get(), i);
+  }
+  for (size_t i = 0; i < 3; i++) {
+    EXPECT_DEATH(bn_assert_fits_in_bytes(x.get(), i), "");
+  }
+}
+
+TEST(BNAssertTest, assert_fits_in_bytes_zero) {
+  bssl::UniquePtr<BIGNUM> x(BN_new());
+  uint8_t input[8];
+  OPENSSL_memset(input, 0, sizeof(input));
+  ASSERT_TRUE(BN_le2bn(input, sizeof(input), x.get()));
+
+  for (size_t i = 0; i < 10; i++) {
+    bn_assert_fits_in_bytes(x.get(), i);
+  }
+}
+
+TEST(BNAssertTest, assert_fits_in_bytes_boundary) {
+  bssl::UniquePtr<BIGNUM> x(BN_new());
+  uint8_t input[8];
+  OPENSSL_memset(input, 0, sizeof(input));
+  for (size_t i = 0; i < sizeof(input); i++) {
+    input[i] = i * (i + 1) & 0xff;
+  }
+  ASSERT_TRUE(BN_le2bn(input, sizeof(input), x.get()));
+  for (size_t i = 8; i < 18; i++) {
+    bn_assert_fits_in_bytes(x.get(), i);
+  }
+  for (size_t i = 0; i < 8; i++) {
+    EXPECT_DEATH(bn_assert_fits_in_bytes(x.get(), i), "");
+  }
+}

crypto/fipsmodule/bn/bytes.c

@@ -210,9 +210,29 @@ void bn_assert_fits_in_bytes(const BIGNUM *bn, size_t num) {
   size_t tot_bytes = bn->width * sizeof(BN_ULONG);
   if (tot_bytes > num) {
     CONSTTIME_DECLASSIFY(bytes + num, tot_bytes - num);
+#ifdef OPENSSL_BIG_ENDIAN
+    int word_idx = num / sizeof(BN_ULONG);
+
+    size_t bytes_to_ignore = num % sizeof(BN_ULONG);
+    if(bytes_to_ignore > 0) {
+      BN_ULONG word = bn->d[word_idx];
+      // bytes in this word at an index higher than `bytes_to_ignore` must be 0.
+      for(size_t i = bytes_to_ignore; i < sizeof(BN_ULONG); i++) {
+        uint8_t byte = (word >> (i * 8)) & 0xff;
+        assert(byte == 0);
+      }
+      word_idx += 1;
+    }
+
+    // subsequent words must all be 0.
+    for(; word_idx < bn->width; word_idx++) {
+      assert(bn->d[word_idx] == 0);
+    }
+#else
     for (size_t i = num; i < tot_bytes; i++) {
       assert(bytes[i] == 0);
     }
+#endif
     (void)bytes;
   }
 }