fix(node): object keys in publicEncrypt (denoland#20128)

Fixes denoland#19935

cli/tests/unit_node/crypto/crypto_cipher_test.ts

@@ -33,6 +33,21 @@ Deno.test({
   },
 });
 
+Deno.test({
+  name: "rsa public encrypt (options) and private decrypt",
+  fn() {
+    const encrypted = crypto.publicEncrypt(
+      { key: Buffer.from(rsaPublicKey) },
+      input,
+    );
+    const decrypted = crypto.privateDecrypt(
+      Buffer.from(rsaPrivateKey),
+      Buffer.from(encrypted),
+    );
+    assertEquals(decrypted, input);
+  },
+});
+
 Deno.test({
   name: "rsa private encrypt and private decrypt",
   fn() {

ext/node/polyfills/internal/crypto/cipher.ts

@@ -13,13 +13,26 @@ import { Buffer } from "node:buffer";
 import { notImplemented } from "ext:deno_node/_utils.ts";
 import type { TransformOptions } from "ext:deno_node/_stream.d.ts";
 import { Transform } from "ext:deno_node/_stream.mjs";
-import { KeyObject } from "./keys.ts";
+import {
+  getArrayBufferOrView,
+  KeyObject,
+} from "ext:deno_node/internal/crypto/keys.ts";
 import type { BufferEncoding } from "ext:deno_node/_global.d.ts";
 import type {
   BinaryLike,
   Encoding,
 } from "ext:deno_node/internal/crypto/types.ts";
 import { getDefaultEncoding } from "ext:deno_node/internal/crypto/util.ts";
+import {
+  isAnyArrayBuffer,
+  isArrayBufferView,
+} from "ext:deno_node/internal/util/types.ts";
+
+function isStringOrBuffer(val) {
+  return typeof val === "string" ||
+    isArrayBufferView(val) ||
+    isAnyArrayBuffer(val);
+}
 
 const { ops, encode } = globalThis.__bootstrap.core;
 
@@ -355,24 +368,51 @@ export function privateEncrypt(
   privateKey: ArrayBufferView | string | KeyObject,
   buffer: ArrayBufferView | string | KeyObject,
 ): Buffer {
+  const { data } = prepareKey(privateKey);
   const padding = privateKey.padding || 1;
-  return ops.op_node_private_encrypt(privateKey, buffer, padding);
+
+  buffer = getArrayBufferOrView(buffer, "buffer");
+  return ops.op_node_private_encrypt(data, buffer, padding);
 }
 
 export function privateDecrypt(
   privateKey: ArrayBufferView | string | KeyObject,
   buffer: ArrayBufferView | string | KeyObject,
 ): Buffer {
+  const { data } = prepareKey(privateKey);
   const padding = privateKey.padding || 1;
-  return ops.op_node_private_decrypt(privateKey, buffer, padding);
+
+  buffer = getArrayBufferOrView(buffer, "buffer");
+  return ops.op_node_private_decrypt(data, buffer, padding);
 }
 
 export function publicEncrypt(
   publicKey: ArrayBufferView | string | KeyObject,
   buffer: ArrayBufferView | string | KeyObject,
 ): Buffer {
+  const { data } = prepareKey(publicKey);
   const padding = publicKey.padding || 1;
-  return ops.op_node_public_encrypt(publicKey, buffer, padding);
+
+  buffer = getArrayBufferOrView(buffer, "buffer");
+  return ops.op_node_public_encrypt(data, buffer, padding);
+}
+
+function prepareKey(key) {
+  // TODO(@littledivy): handle these cases
+  // - node KeyObject
+  // - web CryptoKey
+  if (isStringOrBuffer(key)) {
+    return { data: getArrayBufferOrView(key, "key") };
+  } else if (typeof key == "object") {
+    const { key: data, encoding } = key;
+    if (!isStringOrBuffer(data)) {
+      throw new TypeError("Invalid key type");
+    }
+
+    return { data: getArrayBufferOrView(data, "key", encoding) };
+  }
+
+  throw new TypeError("Invalid key type");
 }
 
 export function publicDecrypt() {

ext/node/polyfills/internal/crypto/keys.ts

@@ -39,7 +39,7 @@ import {
   forgivingBase64UrlEncode as encodeToBase64Url,
 } from "ext:deno_web/00_infra.js";
 
-const getArrayBufferOrView = hideStackFrames(
+export const getArrayBufferOrView = hideStackFrames(
   (
     buffer,
     name,