ed25519 signature malleability test like ed25519-dalek verify_strict

by checking if the signature piece R and public key are small order

src/ballet/ed25519/Local.mk

@@ -1,4 +1,6 @@
 $(call add-hdrs,fd_ed25519.h)
 $(call add-objs,fd_ed25519_fe fd_ed25519_ge fd_ed25519_user,fd_ballet)
 $(call make-unit-test,test_ed25519,test_ed25519,fd_ballet fd_util)
+$(call make-unit-test,test_ed25519_signature_malleability,test_ed25519_signature_malleability,fd_ballet fd_util)
 $(call run-unit-test,test_ed25519,)
+$(call run-unit-test,test_ed25519_signature_malleability,)

src/ballet/ed25519/fd_ed25519_ge.c

@@ -1,10 +1,66 @@
 #include "fd_ed25519_private.h"
 
-#if FD_ED25519_FE_IMPL==0
+#if FD_ED25519_FE_IMPL == 0
 #include "ref/fd_ed25519_ge.c"
-#elif FD_ED25519_FE_IMPL==1
+#elif FD_ED25519_FE_IMPL == 1
 #include "avx/fd_ed25519_ge.c"
 #else
 #error "Unsupported FD_ED25519_FE_IMPL"
 #endif
 
+static inline int
+fd_ed25519_fe_eq( fd_ed25519_fe_t * const fe0,
+                   fd_ed25519_fe_t * const fe1 ) {
+  return ( fe0->limb[ 0 ] == fe1->limb[ 0 ] ) & ( fe0->limb[ 1 ] == fe1->limb[ 1 ] ) &
+         ( fe0->limb[ 2 ] == fe1->limb[ 2 ] ) & ( fe0->limb[ 3 ] == fe1->limb[ 3 ] ) &
+         ( fe0->limb[ 4 ] == fe1->limb[ 4 ] ) & ( fe0->limb[ 5 ] == fe1->limb[ 5 ] ) &
+         ( fe0->limb[ 6 ] == fe1->limb[ 6 ] ) & ( fe0->limb[ 7 ] == fe1->limb[ 7 ] ) &
+         ( fe0->limb[ 8 ] == fe1->limb[ 8 ] ) & ( fe0->limb[ 9 ] == fe1->limb[ 9 ] );
+}
+
+static inline void
+fd_ed25519_ge_p3_mul_by_pow_2( fd_ed25519_ge_p3_t * ret,
+                               fd_ed25519_ge_p3_t * const p,
+                               uint k ) {
+  /* If k is zero then return the original point p. [2^0]P = [1]P = P */
+  if ( FD_UNLIKELY( k == 0 ) ) {
+    *ret = *p;
+    return;
+  }
+  fd_ed25519_ge_p1p1_t r[ 1 ];
+  fd_ed25519_ge_p2_t   s[ 1 ];
+  fd_ed25519_ge_p3_to_p2( s, p );
+  for( uint i = 0; i < ( k - 1 ); i++ ) {
+    fd_ed25519_ge_p2_dbl( r, s );
+    fd_ed25519_ge_p1p1_to_p2( s, r );
+  }
+  fd_ed25519_ge_p2_dbl( r, s );
+  fd_ed25519_ge_p1p1_to_p3( ret, r );
+}
+
+static inline int
+fd_ed25519_ge_p3_is_identity( fd_ed25519_ge_p3_t * const p ) {
+  fd_ed25519_ge_p3_t I[1];
+  fd_ed25519_fe_0( I->X );
+  fd_ed25519_fe_1( I->Y );
+  fd_ed25519_fe_1( I->Z );
+  fd_ed25519_fe_0( I->T );
+
+  fd_ed25519_fe_t cmp[2];
+  fd_ed25519_fe_mul( &cmp[ 0 ], p->X, I->Z );
+  fd_ed25519_fe_mul( &cmp[ 1 ], I->X, p->Z );
+  int x = fd_ed25519_fe_eq( &cmp[ 0 ], &cmp[ 1 ] );
+
+  fd_ed25519_fe_mul( &cmp[ 0 ], p->Y, I->Z );
+  fd_ed25519_fe_mul( &cmp[ 1 ], I->Y, p->Z );
+  int y = fd_ed25519_fe_eq( &cmp[ 0 ], &cmp[ 1 ] );
+
+  return x & y;
+}
+
+int
+fd_ed25519_ge_p3_is_small_order( fd_ed25519_ge_p3_t * const p ) {
+  fd_ed25519_ge_p3_t t[ 1 ];
+  fd_ed25519_ge_p3_mul_by_pow_2( t, p, 3 );
+  return fd_ed25519_ge_p3_is_identity( t );
+}

src/ballet/ed25519/fd_ed25519_private.h

@@ -37,7 +37,7 @@ struct fd_ed25519_ge_p2_private {
   fd_ed25519_fe_t Z[1];
 };
 
-typedef struct fd_ed25519_ge_p2_private fd_ed25519_ge_p2_t; 
+typedef struct fd_ed25519_ge_p2_private fd_ed25519_ge_p2_t;
 
 struct fd_ed25519_ge_p3_private {
   fd_ed25519_fe_t X[1];
@@ -68,6 +68,8 @@ int
 fd_ed25519_ge_frombytes_vartime_2( fd_ed25519_ge_p3_t * h0, uchar const * s0,   /* 32 */
                                    fd_ed25519_ge_p3_t * h1, uchar const * s1 ); /* 32 */
 
+int fd_ed25519_ge_p3_is_small_order(fd_ed25519_ge_p3_t * const p);
+
 static inline fd_ed25519_ge_p2_t *
 fd_ed25519_ge_p2_0( fd_ed25519_ge_p2_t * h ) {
   fd_ed25519_fe_0( h->X );

src/ballet/ed25519/fd_ed25519_user.c

@@ -399,6 +399,8 @@ fd_ed25519_verify( void const *  msg,
      (and hence an inversion) at the end */
   fd_ed25519_ge_p3_t rD[1];
   int err = fd_ed25519_ge_frombytes_vartime_2( A, public_key, rD, r ); if( FD_UNLIKELY( err ) ) return err;
+  if( fd_ed25519_ge_p3_is_small_order(A) )  return FD_ED25519_ERR_PUBKEY;
+  if( fd_ed25519_ge_p3_is_small_order(rD) ) return FD_ED25519_ERR_SIG;
 # else
   int err = fd_ed25519_ge_frombytes_vartime( A, public_key ); if( FD_UNLIKELY( err ) ) return err;
 # endif

src/ballet/ed25519/test_ed25519_signature_malleability.c

@@ -0,0 +1,53 @@
+#include "../fd_ballet.h"
+
+struct verification_test {
+  uchar sig[ 64 ];
+  uchar pub[ 32 ];
+};
+typedef struct verification_test verification_test_t;
+
+FD_IMPORT_BINARY(should_fail_bin, "src/ballet/ed25519/test_ed25519_signature_malleability_should_fail.bin");
+FD_IMPORT_BINARY(should_pass_bin, "src/ballet/ed25519/test_ed25519_signature_malleability_should_pass.bin");
+verification_test_t * const should_fail = ( verification_test_t * const ) should_fail_bin;
+verification_test_t * const should_pass = ( verification_test_t * const ) should_pass_bin;
+
+int
+main( int     argc,
+      char ** argv ) {
+  fd_boot( &argc, &argv );
+  fd_sha512_t _sha[1];
+  fd_sha512_t *sha = fd_sha512_join(fd_sha512_new(_sha));
+  uchar msg[] = "Zcash";
+
+  ulong should_fail_cnt = should_fail_bin_sz/sizeof(verification_test_t);
+  for( ulong i=0UL; i<should_fail_cnt; i++ ) {
+    if( fd_ed25519_verify( msg, 5, should_fail[i].sig, should_fail[i].pub, sha ) == FD_ED25519_SUCCESS ) {
+      FD_LOG_ERR(("FAIL: verify should have failed\n\t"
+                      "index %lu\n\t"
+                      "sig: " FD_LOG_HEX16_FMT "  " FD_LOG_HEX16_FMT "\n\t"
+                      "pub: " FD_LOG_HEX16_FMT,
+              i,
+              FD_LOG_HEX16_FMT_ARGS(should_fail[i].sig),
+              FD_LOG_HEX16_FMT_ARGS(should_fail[i].sig+32),
+              FD_LOG_HEX16_FMT_ARGS(should_fail[i].pub)));
+    }
+  }
+
+  ulong should_pass_cnt = should_pass_bin_sz/sizeof(verification_test_t);
+  for( ulong i=0UL; i<should_pass_cnt; i++ ) {
+    if( fd_ed25519_verify( msg, 5, should_pass[i].sig, should_pass[i].pub, sha ) != FD_ED25519_SUCCESS ) {
+      FD_LOG_ERR(("FAIL: verify should have passed\n\t"
+                  "index %lu\n\t"
+                  "sig: " FD_LOG_HEX16_FMT "  " FD_LOG_HEX16_FMT "\n\t"
+                  "pub: " FD_LOG_HEX16_FMT,
+          i,
+          FD_LOG_HEX16_FMT_ARGS(should_fail[i].sig),
+          FD_LOG_HEX16_FMT_ARGS(should_fail[i].sig+32),
+          FD_LOG_HEX16_FMT_ARGS(should_fail[i].pub)));
+    }
+  }
+
+  FD_LOG_NOTICE(( "pass" ));
+  fd_halt();
+  return 0;
+}