More precise writebarrier for regions

src/coreclr/gc/gcinterface.h

@@ -108,6 +108,12 @@ struct WriteBarrierParameters
     // The new write watch table, if we are using our own write watch
     // implementation. Used for WriteBarrierOp::SwitchToWriteWatch only.
     uint8_t* write_watch_table;
+
+    // mapping table from region index to generation
+    uint8_t* region_to_generation_table;
+
+    // shift count - how many bits to shift right to obtain region index from address
+    uint8_t  region_shr;
 };
 
 struct EtwGCSettingsInfo

src/coreclr/gc/gcpriv.h

@@ -1202,6 +1202,9 @@ enum bookkeeping_element
 #ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
     software_write_watch_table_element,
 #endif
+#ifdef USE_REGIONS
+    region_to_generation_table_element,
+#endif //USE_REGIONS
     seg_mapping_table_element,
 #ifdef BACKGROUND_GC
     mark_array_element,
@@ -1441,6 +1444,12 @@ class gc_heap
     // This relocates the SIP regions and return the next non SIP region.
     PER_HEAP
     heap_segment* relocate_advance_to_non_sip (heap_segment* region);
+
+    PER_HEAP_ISOLATED
+    void verify_region_to_generation_map();
+
+    PER_HEAP_ISOLATED
+    void compute_ephemeral_range();
 #ifdef STRESS_REGIONS
     PER_HEAP
     void pin_by_gc (uint8_t* object);
@@ -3662,6 +3671,17 @@ class gc_heap
     size_t* old_card_survived_per_region;
     PER_HEAP_ISOLATED
     size_t region_count;
+
+    // table mapping region number to generation
+    // there are actually two generation numbers per entry:
+    // - the region's current generation
+    // - the region's planned generation, i.e. after the GC
+    PER_HEAP_ISOLATED
+    uint8_t* map_region_to_generation;
+    // same table as above, but skewed so that we can index
+    // directly with address >> min_segment_size_shr
+    PER_HEAP_ISOLATED
+    uint8_t* map_region_to_generation_skewed;
 #endif //USE_REGIONS
 
 #define max_oom_history_count 4
@@ -3712,7 +3732,13 @@ class gc_heap
     PER_HEAP
     void exit_gc_done_event_lock();
 
-#ifndef USE_REGIONS
+#ifdef USE_REGIONS
+    PER_HEAP_ISOLATED
+    uint8_t*  ephemeral_low;      //lowest ephemeral address
+
+    PER_HEAP_ISOLATED
+    uint8_t*  ephemeral_high;     //highest ephemeral address
+#else //!USE_REGIONS
     PER_HEAP
     uint8_t*  ephemeral_low;      //lowest ephemeral address
 

src/coreclr/vm/amd64/JitHelpers_Fast.asm

@@ -25,6 +25,8 @@ EXTERN  g_ephemeral_high:QWORD
 EXTERN  g_lowest_address:QWORD
 EXTERN  g_highest_address:QWORD
 EXTERN  g_card_table:QWORD
+EXTERN  g_region_shr:BYTE
+EXTERN  g_region_to_generation_table:QWORD
 
 ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
 EXTERN g_card_bundle_table:QWORD
@@ -140,6 +142,31 @@ endif
     align 16
     Exit:
         REPRET
+
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+        NOP_3_BYTE
+
 else
         ; JIT_WriteBarrier_PostGrow64
 
@@ -310,6 +337,55 @@ endif
         cmp     rcx, [g_ephemeral_high]
         jnb     Exit
 
+        ; do the following checks only if we are allowed to trash rax
+        ; otherwise we don't have enough registers
+ifdef FEATURE_USE_SOFTWARE_WRITE_WATCH_FOR_GC_HEAP
+        mov     rax, rcx
+
+        mov     cl, [g_region_shr]
+        test    cl, cl
+        je      SkipCheck
+
+        ; check if the source is in gen 2 - then it's not an ephemeral pointer
+        shr     rax, cl
+        add     rax, [g_region_to_generation_table]
+        cmp     byte ptr [rax], 22h
+        je      Exit
+
+        ; check if the destination happens to be in gen 0
+        mov     rax, rdi
+        shr     rax, cl
+        add     rax, [g_region_to_generation_table]
+        cmp     byte ptr [rax], 0
+        je      Exit
+    SkipCheck:
+
+        ; compute card table bit
+        mov     rcx, rdi
+        mov     al, 1
+        shr     rcx, 8
+        and     cl, 7
+        shl     al, cl
+
+        ; move current rdi value into rcx and then increment the pointers
+        mov     rcx, rdi
+        add     rsi, 8h
+        add     rdi, 8h
+
+        ; Check if we need to update the card table
+        ; Calc pCardByte
+        shr     rcx, 0Bh
+        add     rcx, [g_card_table]
+
+        ; Check if this card table bit is already set
+        test    byte ptr [rcx], al
+        je      SetCardTableBit
+        REPRET
+
+    SetCardTableBit:
+        lock or byte ptr [rcx], al
+else
+
         ; move current rdi value into rcx and then increment the pointers
         mov     rcx, rdi
         add     rsi, 8h
@@ -327,6 +403,7 @@ endif
 
     UpdateCardTable:
         mov     byte ptr [rcx], 0FFh
+endif
 ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
         ; check if we need to update the card bundle table
         ; restore destination address from rdi - rdi has been incremented by 8 already

src/coreclr/vm/amd64/JitHelpers_FastWriteBarriers.asm

@@ -201,6 +201,81 @@ endif
         ret
 LEAF_END_MARKED JIT_WriteBarrier_SVR64, _TEXT
 
+LEAF_ENTRY JIT_WriteBarrier_Region64, _TEXT
+        align 8
+
+        ; Do the move into the GC .  It is correct to take an AV here, the EH code
+        ; figures out that this came from a WriteBarrier and correctly maps it back
+        ; to the managed method which called the WriteBarrier (see setup in
+        ; InitializeExceptionHandling, vm\exceptionhandling.cpp).
+        mov     [rcx], rdx
+
+        mov     r8, rcx
+
+PATCH_LABEL JIT_WriteBarrier_Region64_Patch_Label_RegionToGeneration
+        mov     rax, 0F0F0F0F0F0F0F0F0h
+
+PATCH_LABEL JIT_WriteBarrier_Region64_Patch_Label_RegionShrDest
+        shr     rcx, 16h ; compute region index
+
+        ; Check whether the region we're storing into is gen 0 - nothing to do in this case
+        cmp     byte ptr [rcx + rax], 0
+        jne     NotGen0
+        REPRET
+
+        NOP_2_BYTE ; padding for alignment of constant
+
+    NotGen0:
+PATCH_LABEL JIT_WriteBarrier_Region64_Patch_Label_Lower
+        mov     r9, 0F0F0F0F0F0F0F0F0h
+        cmp     rdx, r9
+        jae     NotLow
+        ret
+    NotLow:
+PATCH_LABEL JIT_WriteBarrier_Region64_Patch_Label_Upper
+        mov     r9, 0F0F0F0F0F0F0F0F0h
+        cmp     rdx, r9
+        jb      NotHigh
+        REPRET
+    NotHigh:
+PATCH_LABEL JIT_WriteBarrier_Region64_Patch_Label_RegionShrSrc
+        shr     rdx, 16h ; compute region index
+        mov     dl, [rdx + rax]
+        cmp     dl, [rcx + rax]
+        jb      isOldToYoung
+        REPRET
+        nop
+
+    IsOldToYoung:
+PATCH_LABEL JIT_WriteBarrier_Region64_Patch_Label_CardTable
+        mov     rax, 0F0F0F0F0F0F0F0F0h
+
+        mov     ecx, r8d
+        shr     r8, 0Bh
+        shr     ecx, 8
+        and     ecx, 7
+        mov     dl, 1
+        shl     dl, cl
+        test    byte ptr [r8 + rax], dl
+        je      UpdateCardTable
+        REPRET
+
+    UpdateCardTable:
+        lock or byte ptr [r8 + rax], dl
+ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+PATCH_LABEL JIT_WriteBarrier_Region64_Patch_Label_CardBundleTable
+        mov     rax, 0F0F0F0F0F0F0F0F0h
+        shr     r8, 0Ah
+        cmp     byte ptr [r8 + rax], 0FFh
+        jne     UpdateCardBundleTable
+        REPRET
+
+    UpdateCardBundleTable:
+        mov     byte ptr [r8 + rax], 0FFh
+endif
+        ret
+LEAF_END_MARKED JIT_WriteBarrier_Region64, _TEXT
+
 endif
 
 
@@ -410,6 +485,92 @@ endif
 LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_SVR64, _TEXT
 
 endif
+
+LEAF_ENTRY JIT_WriteBarrier_WriteWatch_Region64, _TEXT
+        align 8
+
+        ; Do the move into the GC .  It is correct to take an AV here, the EH code
+        ; figures out that this came from a WriteBarrier and correctly maps it back
+        ; to the managed method which called the WriteBarrier (see setup in
+        ; InitializeExceptionHandling, vm\exceptionhandling.cpp).
+        mov     [rcx], rdx
+
+        ; Update the write watch table if necessary
+        mov     rax, rcx
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_Region64_Patch_Label_WriteWatchTable
+        mov     r8, 0F0F0F0F0F0F0F0F0h
+        shr     rax, 0Ch ; SoftwareWriteWatch::AddressToTableByteIndexShift
+        add     rax, r8
+        mov     r8, rcx
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_Region64_Patch_Label_RegionShrDest
+        shr     rcx, 16h ; compute region index
+        cmp     byte ptr [rax], 0h
+        jne     JIT_WriteBarrier_WriteWatch_Region64_Patch_Label_RegionToGeneration
+        mov     byte ptr [rax], 0FFh
+
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_Region64_Patch_Label_RegionToGeneration
+        mov     rax, 0F0F0F0F0F0F0F0F0h
+
+        ; Check whether the region we're storing into is gen 0 - nothing to do in this case
+        cmp     byte ptr [rcx + rax], 0
+        jne     NotGen0
+        REPRET
+
+        NOP_2_BYTE ; padding for alignment of constant
+        NOP_2_BYTE ; padding for alignment of constant
+        NOP_2_BYTE ; padding for alignment of constant
+
+    NotGen0:
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_Region64_Patch_Label_Lower
+        mov     r9, 0F0F0F0F0F0F0F0F0h
+        cmp     rdx, r9
+        jae     NotLow
+        ret
+    NotLow:
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_Region64_Patch_Label_Upper
+        mov     r9, 0F0F0F0F0F0F0F0F0h
+        cmp     rdx, r9
+        jb      NotHigh
+        REPRET
+    NotHigh:
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_Region64_Patch_Label_RegionShrSrc
+        shr     rdx, 16h ; compute region index
+        mov     dl, [rdx + rax]
+        cmp     dl, [rcx + rax]
+        jb      isOldToYoung
+        REPRET
+        nop
+
+    IsOldToYoung:
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_Region64_Patch_Label_CardTable
+        mov     rax, 0F0F0F0F0F0F0F0F0h
+
+        mov     ecx, r8d
+        shr     r8, 0Bh
+        shr     ecx, 8
+        and     ecx, 7
+        mov     dl, 1
+        shl     dl, cl
+        test    byte ptr [r8 + rax], dl
+        je      UpdateCardTable
+        REPRET
+
+    UpdateCardTable:
+        lock or byte ptr [r8 + rax], dl
+ifdef FEATURE_MANUALLY_MANAGED_CARD_BUNDLES
+PATCH_LABEL JIT_WriteBarrier_WriteWatch_Region64_Patch_Label_CardBundleTable
+        mov     rax, 0F0F0F0F0F0F0F0F0h
+        shr     r8, 0Ah
+        cmp     byte ptr [r8 + rax], 0FFh
+        jne     UpdateCardBundleTable
+        REPRET
+
+    UpdateCardBundleTable:
+        mov     byte ptr [r8 + rax], 0FFh
+endif
+        ret
+LEAF_END_MARKED JIT_WriteBarrier_WriteWatch_Region64, _TEXT
+
 endif