[FE 1.0] Check callable reference return type safety during resolution

^KT-51844
^KT-52503 Fixed

compiler/frontend/src/org/jetbrains/kotlin/util/ReenteringLazyValueComputationException.java

@@ -19,8 +19,9 @@
 import com.intellij.openapi.application.Application;
 import com.intellij.openapi.application.ApplicationManager;
 import org.jetbrains.annotations.NotNull;
+import org.jetbrains.kotlin.types.error.LazyWrappedTypeComputationException;
 
-public class ReenteringLazyValueComputationException extends RuntimeException {
+public class ReenteringLazyValueComputationException extends LazyWrappedTypeComputationException {
     public ReenteringLazyValueComputationException() {
     }
 

compiler/resolution/src/org/jetbrains/kotlin/resolve/calls/model/CallableReferencesCandidateFactory.kt

@@ -27,6 +27,7 @@ import org.jetbrains.kotlin.types.*
 import org.jetbrains.kotlin.types.error.ErrorScopeKind
 import org.jetbrains.kotlin.types.error.ErrorUtils
 import org.jetbrains.kotlin.types.error.ErrorTypeKind
+import org.jetbrains.kotlin.types.error.LazyWrappedTypeComputationException
 import org.jetbrains.kotlin.types.expressions.CoercionStrategy
 import org.jetbrains.kotlin.types.typeUtil.isUnit
 import org.jetbrains.kotlin.utils.SmartList
@@ -239,8 +240,15 @@ class CallableReferencesCandidateFactory(
         // lower(Unit!) = Unit
         val returnExpectedType = inputOutputTypes.outputType
 
+        fun isReturnTypeNonUnitSafe(): Boolean =
+            try {
+                descriptor.returnType?.isUnit() == false
+            } catch (e: LazyWrappedTypeComputationException) {
+                false
+            }
+
         val coercion =
-            if (returnExpectedType.isUnit() && descriptor.returnType?.isUnit() == false)
+            if (returnExpectedType.isUnit() && isReturnTypeNonUnitSafe())
                 CoercionStrategy.COERCION_TO_UNIT
             else
                 CoercionStrategy.NO_COERCION

compiler/testData/codegen/box/callableReference/kt51844.kt

@@ -0,0 +1,15 @@
+// WITH_STDLIB
+
+abstract class Foo {
+    abstract fun contains(x: Int);
+}
+
+// ERROR: Type checking has run into a recursive problem. Easiest workaround: specify types of your declarations explicitly
+fun Foo.contains(vararg xs: Int) = xs.forEach(this::contains)
+
+fun box(): String {
+    object : Foo() {
+        override fun contains(x: Int) {}
+    }.contains(1)
+    return "OK"
+}

compiler/testData/diagnostics/tests/callableReference/kt52503.kt

@@ -0,0 +1,16 @@
+// FIR_IDENTICAL
+// WITH_STDLIB
+
+abstract class Foo
+
+object A {
+    fun Foo.contains(vararg xs: Int) = // 1
+        xs.forEach(this::<!EXTENSION_IN_CLASS_REFERENCE_NOT_ALLOWED!>contains<!>) // resolved to (2) in 1.7.0-RC, should be error "Type checking has run into a recursive problem"
+}
+
+fun Any.contains(vararg xs: Int) {} // 2
+
+fun box(): String {
+    object : Foo() {}.contains(1)
+    return "OK"
+}

compiler/testData/diagnostics/tests/callableReference/kt52503.txt

@@ -0,0 +1,19 @@
+package
+
+public fun box(): kotlin.String
+public fun kotlin.Any.contains(/*0*/ vararg xs: kotlin.Int /*kotlin.IntArray*/): kotlin.Unit
+
+public object A {
+    private constructor A()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+    public final fun Foo.contains(/*0*/ vararg xs: kotlin.Int /*kotlin.IntArray*/): kotlin.Unit
+}
+
+public abstract class Foo {
+    public constructor Foo()
+    public open override /*1*/ /*fake_override*/ fun equals(/*0*/ other: kotlin.Any?): kotlin.Boolean
+    public open override /*1*/ /*fake_override*/ fun hashCode(): kotlin.Int
+    public open override /*1*/ /*fake_override*/ fun toString(): kotlin.String
+}

compiler/testData/diagnostics/tests/platformTypes/kt50877.kt

@@ -1,3 +1,4 @@
+// FIR_IDENTICAL
 // FULL_JDK
 // WITH_STDLIB
 

core/descriptors/src/org/jetbrains/kotlin/types/error/LazyWrappedTypeComputationException.kt

@@ -0,0 +1,8 @@
+/*
+ * Copyright 2010-2022 JetBrains s.r.o. and Kotlin Programming Language contributors.
+ * Use of this source code is governed by the Apache 2.0 license that can be found in the license/LICENSE.txt file.
+ */
+
+package org.jetbrains.kotlin.types.error
+
+abstract class LazyWrappedTypeComputationException : RuntimeException()