FIX: RSA binary input values were expecting, that series are always a…

…t its head
Oldes · Feb 18, 2022 · 60bfc8b · 60bfc8b
1 parent f0d33e5
commit 60bfc8b
Show file tree

Hide file tree

Showing 2 changed files with 41 additions and 20 deletions.
diff --git a/src/core/n-crypt.c b/src/core/n-crypt.c
@@ -245,12 +245,12 @@ static mbedtls_ctr_drbg_context ctr_drbg;
 #ifndef INCLUDE_RSA
 	Trap0(RE_FEATURE_NA);
 #else
-	REBSER *n       = VAL_SERIES(D_ARG(1));
-	REBSER *e       = VAL_SERIES(D_ARG(2));
+	REBVAL *n       = D_ARG(1);
+	REBVAL *e       = D_ARG(2);
 	REBOOL  ref_private =        D_REF(3);
-	REBSER *d       = VAL_SERIES(D_ARG(4));
-	REBSER *p       = VAL_SERIES(D_ARG(5));
-	REBSER *q       = VAL_SERIES(D_ARG(6));
+	REBVAL *d       = D_ARG(4);
+	REBVAL *p       = D_ARG(5);
+	REBVAL *q       = D_ARG(6);
 
 	int err = 0;
 	REBVAL *ret = D_RET;
@@ -264,11 +264,11 @@ static mbedtls_ctr_drbg_context ctr_drbg;
 	if (ref_private) {
 		err = mbedtls_rsa_import_raw(
 			rsa_ctx,
-			BIN_DATA(n), BIN_LEN(n),
-			BIN_DATA(p), BIN_LEN(p),
-			BIN_DATA(q), BIN_LEN(q),
-			BIN_DATA(d), BIN_LEN(d),
-			BIN_DATA(e), BIN_LEN(e)
+			VAL_BIN_AT(n), VAL_LEN(n),
+			VAL_BIN_AT(p), VAL_LEN(p),
+			VAL_BIN_AT(q), VAL_LEN(q),
+			VAL_BIN_AT(d), VAL_LEN(d),
+			VAL_BIN_AT(e), VAL_LEN(e)
 		);
 		if (err != 0 
 			|| mbedtls_rsa_complete(rsa_ctx) != 0
@@ -277,11 +277,11 @@ static mbedtls_ctr_drbg_context ctr_drbg;
 	} else {
 		err = mbedtls_rsa_import_raw(
 			rsa_ctx,
-			BIN_DATA(n), BIN_LEN(n),
+			VAL_BIN_AT(n), VAL_LEN(n),
 			NULL, 0,
 			NULL, 0,
 			NULL, 0,
-			BIN_DATA(e), BIN_LEN(e)
+			VAL_BIN_AT(e), VAL_LEN(e)
 		);
 		if (err != 0 
 			|| mbedtls_rsa_complete(rsa_ctx) != 0
@@ -327,7 +327,7 @@ static int myrand(void *rng_state, unsigned char *output, size_t len)
 //		/verify   "Use public key to verify signed data (returns TRUE or FALSE)"
 //		 signature [binary!] "Result of the /sign call"
 //		/hash     "Signature's message digest algorithm"
-//		 algorithm [word!] "Default value is SHA256"
+//		 algorithm [word! none!]
 //  ]
 ***********************************************************************/
 {
@@ -349,6 +349,7 @@ static int myrand(void *rng_state, unsigned char *output, size_t len)
 	REBYTE  *inBinary;
 	REBYTE  *outBinary;
 	REBYTE   hash[64];
+	REBCNT   hashSym;
 	REBCNT   inBytes;
 	REBCNT   outBytes;
 	REBINT   err = 0;
@@ -381,28 +382,31 @@ static int myrand(void *rng_state, unsigned char *output, size_t len)
 		return R_NONE;
 	}
 
-	data = VAL_SERIES(val_data);
-	inBinary = BIN_DATA(data);
-	inBytes = BIN_LEN(data);
+	data     = VAL_SERIES(val_data);
+	inBinary = VAL_BIN_AT(val_data);
+	inBytes  = VAL_LEN(val_data);
 
 	if (refVerify || refSign) {
-		if (IS_NONE(val_hash)) {
+		if (refHash && IS_NONE(val_hash)) {
+			// use none if really requested
 			md_alg = MBEDTLS_MD_NONE;
 		}
 		else {
+			// if /hash was not used, make default to SHA256
+			hashSym = IS_NONE(val_hash) ? SYM_SHA256 : VAL_WORD_CANON(val_hash);
 			// count message digest off the input data
-			if (Message_Digest(hash, inBinary, inBytes, VAL_WORD_CANON(val_hash), &inBytes)) {
+			if (Message_Digest(hash, inBinary, inBytes, hashSym, &inBytes)) {
 				// map Rebol word to mbedtls_md_type_t (expets that have same order!)
 				// no need to test a range as only known will pass above run
-				md_alg = VAL_WORD_CANON(val_hash) - SYM_MD5 + 1;
+				md_alg = hashSym - SYM_MD5 + 1;
 				inBinary = hash;
 			}
 			else {
 				return R_NONE;
 			}
 		}
 		if (refVerify) {
-			err = mbedtls_rsa_rsassa_pkcs1_v15_verify(rsa, md_alg, inBytes, inBinary, VAL_BIN(val_sign));
+			err = mbedtls_rsa_rsassa_pkcs1_v15_verify(rsa, md_alg, inBytes, inBinary, VAL_BIN_AT(val_sign));
 			return (err == 0) ? R_TRUE : R_FALSE;
 		}
 	}

diff --git a/src/tests/units/rsa-test.r3 b/src/tests/units/rsa-test.r3
@@ -103,6 +103,23 @@ Rebol [
 		; released handle is now unusable:
 		--assert error? try [rsa/verify/hash key-pub bin-data signature 'SHA512]
 
+	--test-- "Test input where it's not exactly at its head"
+		key-pub: rsa-init
+			at #{0BADCAFE
+			D2FC7B6A0A1E6C67104AEB8F88B257669B4DF679DDAD099B5C4A6CD9A88015B5
+			A133BF0B856C7871B6DF000B554FCEB3C2ED512BB68F145C6E8434752FAB52A1
+			CFC124408F79B58A4578C16428855789F7A249E384CB2D9FAE2D67FD96FB926C
+			198E077399FDC815C0AF097DDE5AADEFF44DE70E827F4878432439BFEEB96068
+			D0474FC50D6D90BF3A98DFAF1040C89C02D692AB3B3C2896609D86FD73B774CE
+			0740647CEEEAA310BD12F985A8EB9F59FDD426CEA5B2120F4F2A34BCAB764B7E
+			6C54D6840238BCC40587A59E66ED1F33894577635C470AF75CF92C20D1DA43E1
+			BFC419E222A6F0D0BB358C5E38F9CB050AEAFE904814F1AC1AA49CCA9EA0CA83
+			} 5	#{010001}
+		--assert rsa/verify key-pub bin-data sign-hash
+		bin-data:  insert bin-data  #{0BADCAFE}
+		sign-hash: insert sign-hash #{0BADCAFE}
+		--assert rsa/verify key-pub bin-data sign-hash
+
 ===end-group===