Inliner: Extend IL limit for profiled call-sites, allow inlining for switches.

src/coreclr/jit/block.cpp

@@ -1431,14 +1431,15 @@ BasicBlock* Compiler::bbNewBasicBlock(BBjumpKinds jumpKind)
     /* Give the block a number, set the ancestor count and weight */
 
     ++fgBBcount;
+    ++fgBBNumMax;
 
     if (compIsForInlining())
     {
         block->bbNum = ++impInlineInfo->InlinerCompiler->fgBBNumMax;
     }
     else
     {
-        block->bbNum = ++fgBBNumMax;
+        block->bbNum = fgBBNumMax;
     }
 
     if (compRationalIRForm)

src/coreclr/jit/compiler.cpp

@@ -6272,6 +6272,9 @@ int Compiler::compCompileHelper(CORINFO_MODULE_HANDLE classPtr,
         // a potential inline candidate.
         InlineResult prejitResult(this, methodHnd, "prejit");
 
+        // Profile data allows us to avoid early "too many IL bytes" outs.
+        prejitResult.NoteBool(InlineObservation::CALLSITE_HAS_PROFILE, fgHaveSufficientProfileData());
+
         // Do the initial inline screen.
         impCanInlineIL(methodHnd, methodInfo, forceInline, &prejitResult);
 

src/coreclr/jit/compiler.h

@@ -5814,6 +5814,7 @@ class Compiler
     void WalkSpanningTree(SpanningTreeVisitor* visitor);
     void fgSetProfileWeight(BasicBlock* block, BasicBlock::weight_t weight);
     void fgApplyProfileScale();
+    bool fgHaveSufficientProfileData();
 
     // fgIsUsingProfileWeights - returns true if we have real profile data for this method
     //                           or if we have some fake profile data for the stress mode

src/coreclr/jit/fgbasic.cpp

@@ -818,8 +818,24 @@ class FgStack
             return false;
         }
         const unsigned argNum = value - SLOT_ARGUMENT;
-        assert(argNum < info->argCnt);
-        return info->inlArgInfo[argNum].argIsInvariant;
+        if (argNum < info->argCnt)
+        {
+            return info->inlArgInfo[argNum].argIsInvariant;
+        }
+        return false;
+    }
+    static bool IsExactArgument(FgSlot value, InlineInfo* info)
+    {
+        if ((info == nullptr) || !IsArgument(value))
+        {
+            return false;
+        }
+        const unsigned argNum = value - SLOT_ARGUMENT;
+        if (argNum < info->argCnt)
+        {
+            return info->inlArgInfo[argNum].argIsExact;
+        }
+        return false;
     }
     static unsigned SlotTypeToArgNum(FgSlot value)
     {
@@ -876,6 +892,10 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
 
     if (makeInlineObservations)
     {
+        // Set default values for profile (to avoid NoteFailed in CALLEE_IL_CODE_SIZE's handler)
+        // these will be overridden later.
+        compInlineResult->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE, true);
+        compInlineResult->NoteDouble(InlineObservation::CALLSITE_PROFILE_FREQUENCY, 1.0);
         // Observe force inline state and code size.
         compInlineResult->NoteBool(InlineObservation::CALLEE_IS_FORCE_INLINE, isForceInline);
         compInlineResult->NoteInt(InlineObservation::CALLEE_IL_CODE_SIZE, codeSize);
@@ -1031,7 +1051,8 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
                 if (makeInlineObservations)
                 {
                     FgStack::FgSlot slot = pushedStack.Top();
-                    if (FgStack::IsConstantOrConstArg(slot, impInlineInfo))
+                    if (FgStack::IsConstantOrConstArg(slot, impInlineInfo) ||
+                        FgStack::IsExactArgument(slot, impInlineInfo))
                     {
                         compInlineResult->Note(InlineObservation::CALLSITE_FOLDABLE_EXPR_UN);
                         handled = true; // and keep argument in the pushedStack
@@ -1338,44 +1359,59 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
                     FgStack::FgSlot arg0 = pushedStack.Top(1);
                     FgStack::FgSlot arg1 = pushedStack.Top(0);
 
-                    if ((FgStack::IsConstant(arg0) && FgStack::IsConstArgument(arg1, impInlineInfo)) ||
-                        (FgStack::IsConstant(arg1) && FgStack::IsConstArgument(arg0, impInlineInfo)) ||
-                        (FgStack::IsConstArgument(arg0, impInlineInfo) &&
-                         FgStack::IsConstArgument(arg1, impInlineInfo)))
+                    // Const op ConstArg -> ConstArg
+                    if (FgStack::IsConstant(arg0) && FgStack::IsConstArgument(arg1, impInlineInfo))
                     {
                         // keep stack unchanged
                         handled = true;
                         compInlineResult->Note(InlineObservation::CALLSITE_FOLDABLE_EXPR);
                     }
-                    if ((FgStack::IsConstant(arg0) && FgStack::IsConstant(arg1)) ||
-                        (FgStack::IsConstant(arg1) && FgStack::IsConstant(arg0)))
+                    // ConstArg op Const    -> ConstArg
+                    // ConstArg op ConstArg -> ConstArg
+                    else if (FgStack::IsConstArgument(arg0, impInlineInfo) &&
+                             FgStack::IsConstantOrConstArg(arg1, impInlineInfo))
+                    {
+                        if (FgStack::IsConstant(arg1))
+                        {
+                            pushedStack.Push(arg0);
+                        }
+                        handled = true;
+                        compInlineResult->Note(InlineObservation::CALLSITE_FOLDABLE_EXPR);
+                    }
+                    // Const op Const -> Const
+                    else if (FgStack::IsConstant(arg0) && FgStack::IsConstant(arg1))
                     {
                         // both are constants, but we're mostly interested in cases where a const arg leads to
                         // a foldable expression.
                         handled = true;
                     }
+                    // Arg op ConstArg
+                    // Arg op Const
                     else if (FgStack::IsArgument(arg0) && FgStack::IsConstantOrConstArg(arg1, impInlineInfo))
                     {
                         // "Arg op CNS" --> keep arg0 in the stack for the next ops
+                        pushedStack.Push(arg0);
                         handled = true;
                         compInlineResult->Note(InlineObservation::CALLEE_BINARY_EXRP_WITH_CNS);
                     }
+                    // ConstArg op Arg
+                    // Const    op Arg
                     else if (FgStack::IsArgument(arg1) && FgStack::IsConstantOrConstArg(arg0, impInlineInfo))
                     {
                         // "CNS op ARG" --> keep arg1 in the stack for the next ops
-                        pushedStack.Push(arg1);
                         handled = true;
                         compInlineResult->Note(InlineObservation::CALLEE_BINARY_EXRP_WITH_CNS);
                     }
-
+                    // X / ConstArg
+                    // X % ConstArg
                     if (FgStack::IsConstArgument(arg1, impInlineInfo))
                     {
-                        // Special case: "X / ConstArg" or "X % ConstArg"
                         if ((opcode == CEE_DIV) || (opcode == CEE_DIV_UN) || (opcode == CEE_REM) ||
                             (opcode == CEE_REM_UN))
                         {
                             compInlineResult->Note(InlineObservation::CALLSITE_DIV_BY_CNS);
                         }
+                        pushedStack.Push(arg0);
                         handled = true;
                     }
                 }
@@ -1583,6 +1619,10 @@ void Compiler::fgFindJumpTargets(const BYTE* codeAddr, IL_OFFSET codeSize, Fixed
                 if (makeInlineObservations)
                 {
                     compInlineResult->Note(InlineObservation::CALLEE_HAS_SWITCH);
+                    if (FgStack::IsConstantOrConstArg(pushedStack.Top(), impInlineInfo))
+                    {
+                        compInlineResult->Note(InlineObservation::CALLSITE_FOLDABLE_SWITCH);
+                    }
 
                     // Fail fast, if we're inlining and can't handle this.
                     if (isInlining && compInlineResult->IsFailure())

src/coreclr/jit/fgprofile.cpp

@@ -47,6 +47,31 @@ bool Compiler::fgHaveProfileData()
     return (fgPgoSchema != nullptr);
 }
 
+//------------------------------------------------------------------------
+// fgHaveSufficientProfileData: check if profile data is available
+//   and is sufficient enough to be trustful.
+//
+// Returns:
+//   true if so
+//
+// Note:
+//   See notes for fgHaveProfileData.
+//
+bool Compiler::fgHaveSufficientProfileData()
+{
+    if (!fgHaveProfileData())
+    {
+        return false;
+    }
+
+    if ((fgFirstBB != nullptr) && (fgPgoSource == ICorJitInfo::PgoSource::Static))
+    {
+        const BasicBlock::weight_t sufficientSamples = 1000;
+        return fgFirstBB->bbWeight > sufficientSamples;
+    }
+    return true;
+}
+
 //------------------------------------------------------------------------
 // fgApplyProfileScale: scale inlinee counts by appropriate scale factor
 //

src/coreclr/jit/importer.cpp

@@ -13258,8 +13258,6 @@ void Compiler::impImportBlockCode(BasicBlock* block)
                 goto COND_JUMP;
 
             case CEE_SWITCH:
-                assert(!compIsForInlining());
-
                 if (tiVerificationNeeded)
                 {
                     Verify(impStackTop().seTypeInfo.IsType(TI_INT), "Bad switch val");
@@ -19071,33 +19069,30 @@ void Compiler::impMakeDiscretionaryInlineObservations(InlineInfo* pInlineInfo, I
     inlineResult->NoteInt(InlineObservation::CALLSITE_FREQUENCY, static_cast<int>(frequency));
     inlineResult->NoteInt(InlineObservation::CALLSITE_WEIGHT, (int)(weight));
 
+    bool   hasProfile  = false;
+    double profileFreq = 0.0;
+
     // If the call site has profile data, report the relative frequency of the site.
     //
-    if ((pInlineInfo != nullptr) && rootCompiler->fgHaveProfileData() && pInlineInfo->iciBlock->hasProfileWeight())
+    if ((pInlineInfo != nullptr) && rootCompiler->fgHaveSufficientProfileData())
     {
-        BasicBlock::weight_t callSiteWeight = pInlineInfo->iciBlock->bbWeight;
-        BasicBlock::weight_t entryWeight    = rootCompiler->fgFirstBB->bbWeight;
-        BasicBlock::weight_t profileFreq    = entryWeight == 0.0f ? 0.0f : callSiteWeight / entryWeight;
+        const BasicBlock::weight_t callSiteWeight = pInlineInfo->iciBlock->bbWeight;
+        const BasicBlock::weight_t entryWeight    = rootCompiler->fgFirstBB->bbWeight;
+        profileFreq                               = entryWeight == 0.0f ? 0.0 : callSiteWeight / entryWeight;
+        hasProfile                                = true;
 
         assert(callSiteWeight >= 0);
         assert(entryWeight >= 0);
-
-        BasicBlock::weight_t sufficientSamples = 1000.0f;
-
-        if (!rootCompiler->opts.jitFlags->IsSet(JitFlags::JIT_FLAG_PREJIT) ||
-            ((callSiteWeight + entryWeight) > sufficientSamples))
-        {
-            // Let's not report profiles for methods with insufficient samples during prejitting.
-            inlineResult->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE, true);
-            inlineResult->NoteDouble(InlineObservation::CALLSITE_PROFILE_FREQUENCY, profileFreq);
-        }
     }
-    else if ((pInlineInfo == nullptr) && rootCompiler->fgHaveProfileData())
+    else if (pInlineInfo == nullptr)
     {
         // Simulate a hot callsite for PrejitRoot mode.
-        inlineResult->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE, true);
-        inlineResult->NoteDouble(InlineObservation::CALLSITE_PROFILE_FREQUENCY, 1.0);
+        hasProfile  = true;
+        profileFreq = 1.0;
     }
+
+    inlineResult->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE, hasProfile);
+    inlineResult->NoteDouble(InlineObservation::CALLSITE_PROFILE_FREQUENCY, profileFreq);
 }
 
 /*****************************************************************************
@@ -19243,6 +19238,10 @@ void Compiler::impCheckCanInline(GenTreeCall*           call,
                 goto _exit;
             }
 
+            // Profile data allows us to avoid early "too many IL bytes" outs.
+            pParam->result->NoteBool(InlineObservation::CALLSITE_HAS_PROFILE,
+                                     pParam->pThis->fgHaveSufficientProfileData());
+
             bool forceInline;
             forceInline = !!(pParam->methAttr & CORINFO_FLG_FORCEINLINE);
 
@@ -19465,6 +19464,10 @@ void Compiler::impInlineRecordArgInfo(InlineInfo*   pInlineInfo,
         }
     }
 
+    bool isExact              = false;
+    bool isNonNull            = false;
+    inlCurArgInfo->argIsExact = (gtGetClassHandle(curArgVal, &isExact, &isNonNull) != NO_CLASS_HANDLE) && isExact;
+
     // If the arg is a local that is address-taken, we can't safely
     // directly substitute it into the inlinee.
     //

src/coreclr/jit/inline.def

@@ -182,6 +182,7 @@ INLINE_OBSERVATION(FOLDABLE_INTRINSIC,        int,    "foldable intrinsic",
 INLINE_OBSERVATION(FOLDABLE_EXPR,             int,    "foldable binary expression",           INFORMATION, CALLSITE)
 INLINE_OBSERVATION(FOLDABLE_EXPR_UN,          int,    "foldable unary expression",            INFORMATION, CALLSITE)
 INLINE_OBSERVATION(FOLDABLE_BRANCH,           int,    "foldable branch",                      INFORMATION, CALLSITE)
+INLINE_OBSERVATION(FOLDABLE_SWITCH,           int,    "foldable switch",                      INFORMATION, CALLSITE)
 INLINE_OBSERVATION(DIV_BY_CNS,                int,    "dividy by const",                      INFORMATION, CALLSITE)
 INLINE_OBSERVATION(CONSTANT_ARG_FEEDS_TEST,   bool,   "constant argument feeds test",         INFORMATION, CALLSITE)
 INLINE_OBSERVATION(DEPTH,                     int,    "depth",                                INFORMATION, CALLSITE)

src/coreclr/jit/inline.h

@@ -610,6 +610,7 @@ struct InlArgInfo
     unsigned argHasStargOp : 1;           // Is there STARG(s) operation on this argument?
     unsigned argIsByRefToStructLocal : 1; // Is this arg an address of a struct local or a normed struct local or a
                                           // field in them?
+    unsigned argIsExact : 1;              // Is this arg of an exact class?
 };
 
 // InlLclVarInfo describes inline candidate argument and local variable properties.