The page write concurrently with SegmentMerge/Split/MergeDelta may not get removed from the PageStorage

[JaySon-Huang]

Bug Report

Please answer these questions before submitting your issue. Thanks!

1. Minimal reproduce step (Required)

TEST_F(SegmentOperationTest, Issue5570Case2)
try
{
    {
        SegmentTestOptions options;
        // a smaller pack rows for logical split
        options.db_settings.dt_segment_stable_pack_rows = 100;
        reloadWithOptions(options);
    }

    writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 100);
    writeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, 100);
    flushSegmentCache(DELTA_MERGE_FIRST_SEGMENT_ID);
    mergeSegmentDelta(DELTA_MERGE_FIRST_SEGMENT_ID);

    FailPointHelper::enableFailPoint(FailPoints::force_segment_logical_split);
    auto new_seg_id_opt = splitSegment(DELTA_MERGE_FIRST_SEGMENT_ID);
    ASSERT_TRUE(new_seg_id_opt.has_value());
    auto new_seg_id = new_seg_id_opt.value();

    const auto storage_pool = db_context->getGlobalContext().getGlobalStoragePool();
    for (size_t round = 0; round < 50; ++round)
    {
        LOG_DEBUG(log, "beginSegmentMerge");

        // Start a segment merge and suspend it before applyMerge
        auto sp_seg_merge_apply = SyncPointCtl::enableInScope("before_Segment::applyMerge");
        auto th_seg_merge = std::async([&]() {
            mergeSegment(DELTA_MERGE_FIRST_SEGMENT_ID, new_seg_id, true);
        });
        sp_seg_merge_apply.waitAndPause();
        LOG_DEBUG(log, "pausedBeforeApplyMerge");

        // flushed pack
        writeSegment(new_seg_id, 100);
        // flushSegmentCache(new_seg_id); // do not flush

        // Finish the segment merge
        LOG_DEBUG(log, "continueApplyMerge");
        sp_seg_merge_apply.next();
        th_seg_merge.wait();
        LOG_DEBUG(log, "finishApplyMerge");

        // logical split
        FailPointHelper::enableFailPoint(FailPoints::try_segment_logical_split);
        auto new_seg_id2_opt = splitSegment(DELTA_MERGE_FIRST_SEGMENT_ID);
        ASSERT_TRUE(new_seg_id2_opt.has_value());
        new_seg_id = new_seg_id2_opt.value();

        const auto file_usage = storage_pool->getLogFileUsage();
        LOG_DEBUG(log, "log valid size: {}", file_usage.total_valid_size);
    }
    for (const auto & [seg_id, seg] : segments)
    {
        UNUSED(seg);
        deleteRangeSegment(seg_id);
        flushSegmentCache(seg_id);
        mergeSegmentDelta(seg_id);
    }
    storage_pool->gc();
    const auto file_usage = storage_pool->getLogFileUsage();
    LOG_DEBUG(log, "all removed, file usage: {}", file_usage.total_valid_size); // should be 0
}
CATCH

2. What did you expect to see? (Required)

After the test run, we call delete range and delta merge for all segments, the valid size of the log PageStorage should be zero.

3. What did you see instead (Required)

The valid size of log PageStorage is not zero

The root cause is in applyMergeDelta/applyMerge/applySplit, we create ref page for all pages that is not exists in the snapshot, but only remove the original page that is flushed into disk.

4. What is your TiFlash version? (Required)

master

[JaySon-Huang]

Introduced by #3943 since 6.0.0