fix: type check for error instance when used toThrowError method

CHANGELOG.md

@@ -20,6 +20,7 @@
 - `[jest-leak-detector]` Make leak-detector more aggressive when running GC ([#14526](https://github.com/jestjs/jest/pull/14526))
 - `[jest-util]` Make sure `isInteractive` works in a browser ([#14552](https://github.com/jestjs/jest/pull/14552))
 - `[pretty-format]` [**BREAKING**] Print `ArrayBuffer` and `DataView` correctly ([#14290](https://github.com/facebook/jest/pull/14290))
+- `[expect]` Check error instance type for `toThrow/toThrowError` ([#14576](https://github.com/jestjs/jest/pull/14576))
 
 ### Performance
 

packages/expect/src/__tests__/toThrowMatchers.test.ts

@@ -169,10 +169,16 @@ describe.each(['toThrowError', 'toThrow'] as const)('%s', toThrow => {
       jestExpect(() => {
         throw new Err();
       })[toThrow](CustomError);
+      jestExpect(() => {
+        throw new SubErr();
+      })[toThrow](new SubErr());
       jestExpect(() => {
         throw new Err();
       }).not[toThrow](Err2);
       jestExpect(() => {}).not[toThrow](Err);
+      jestExpect(() => {
+        throw new SubErr();
+      }).not[toThrow](new SubSubErr());
     });
 
     test('did not throw at all', () => {

packages/expect/src/toThrowMatchers.ts

@@ -228,10 +228,17 @@ const toThrowExpectedObject = (
   const expectedMessageAndCause = createMessageAndCause(expected);
   const thrownMessageAndCause =
     thrown === null ? null : createMessageAndCause(thrown.value);
+  const isCompareErrorInstance = thrown?.isError && expected instanceof Error;
+  const isExpectedCustomErrorInstance =
+    expected.constructor.name !== Error.name;
+
   const pass =
     thrown !== null &&
     thrown.message === expected.message &&
-    thrownMessageAndCause === expectedMessageAndCause;
+    thrownMessageAndCause === expectedMessageAndCause &&
+    (!isCompareErrorInstance ||
+      !isExpectedCustomErrorInstance ||
+      thrown.value instanceof expected.constructor);
 
   const message = pass
     ? () =>