Skip to content

Commit

Permalink
DDL: Skip collecting TiFlash status when TiFlash is down (#40872) (#4…
Browse files Browse the repository at this point in the history
  • Loading branch information
ti-chi-bot authored Aug 16, 2023
1 parent adf6de5 commit a742996
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 1 deletion.
8 changes: 8 additions & 0 deletions ddl/ddl_tiflash_api.go
Original file line number Diff line number Diff line change
Expand Up @@ -424,6 +424,14 @@ func (d *ddl) refreshTiFlashTicker(ctx sessionctx.Context, pollTiFlashContext *T
return err
}
}

failpoint.Inject("OneTiFlashStoreDown", func() {
for storeID, store := range pollTiFlashContext.TiFlashStores {
store.Store.StateName = "Down"
pollTiFlashContext.TiFlashStores[storeID] = store
break
}
})
pollTiFlashContext.PollCounter++

// Start to process every table.
Expand Down
20 changes: 20 additions & 0 deletions ddl/ddl_tiflash_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -1299,3 +1299,23 @@ func TestTiFlashAvailableAfterAddPartition(t *testing.T) {
require.NotNil(t, pi)
require.Equal(t, len(pi.Definitions), 2)
}

func TestTiFlashAvailableAfterDownOneStore(t *testing.T) {
s, teardown := createTiFlashContext(t)
defer teardown()
tk := testkit.NewTestKit(t, s.store)

tk.MustExec("use test")
tk.MustExec("drop table if exists ddltiflash")
tk.MustExec("create table ddltiflash(z int) PARTITION BY RANGE(z) (PARTITION p0 VALUES LESS THAN (10))")
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/ddl/OneTiFlashStoreDown", `return`))
require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/domain/infosync/OneTiFlashStoreDown", `return`))
defer func() {
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/ddl/OneTiFlashStoreDown"))
require.NoError(t, failpoint.Disable("github.com/pingcap/tidb/domain/infosync/OneTiFlashStoreDown"))
}()

tk.MustExec("alter table ddltiflash set tiflash replica 1")
time.Sleep(ddl.PollTiFlashInterval * RoundToBeAvailable * 3)
CheckTableAvailable(s.dom, t, 1, []string{})
}
12 changes: 11 additions & 1 deletion domain/infosync/tiflash_manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ import (

"github.com/gorilla/mux"
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/tidb/ddl/placement"
"github.com/pingcap/tidb/store/helper"
"github.com/pingcap/tidb/tablecodec"
Expand Down Expand Up @@ -93,10 +94,19 @@ func getTiFlashPeerWithoutLagCount(tiFlashStores map[int64]helper.StoreStat, tab
for _, store := range tiFlashStores {
regionReplica := make(map[int64]int)
err := helper.CollectTiFlashStatus(store.Store.StatusAddress, tableID, &regionReplica)
failpoint.Inject("OneTiFlashStoreDown", func() {
if store.Store.StateName == "Down" {
err = errors.New("mock TiFlasah down")
}
})
if err != nil {
logutil.BgLogger().Error("Fail to get peer status from TiFlash.",
zap.Int64("tableID", tableID))
return 0, err
// Just skip down or offline or tomestone stores, because PD will migrate regions from these stores.
if store.Store.StateName == "Up" || store.Store.StateName == "Disconnected" {
return 0, err
}
continue
}
flashPeerCount += len(regionReplica)
}
Expand Down

0 comments on commit a742996

Please sign in to comment.