Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cmd: hide --sort-dir in changefeed command. (deprecated warning exists) (#1784) #1794

Merged
merged 4 commits into from
May 19, 2021
Merged

cmd: hide --sort-dir in changefeed command. (deprecated warning exists) (#1784) #1794

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
3782274
cmd: hide --sort-dir in changefeed command. (deprecated warning exists)
amyangfei May 17, 2021
9ebcc08
Merge branch 'release-4.0' into cherry-pick-1784-to-release-4.0
ti-chi-bot May 19, 2021
e72abd7
Merge branch 'release-4.0' into cherry-pick-1784-to-release-4.0
ti-chi-bot May 19, 2021
6b9951e
Merge branch 'release-4.0' into cherry-pick-1784-to-release-4.0
ti-chi-bot May 19, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion cdc/model/changefeed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,8 @@ type ChangeFeedInfo struct {
// used for admin job notification, trigger watch event in capture
AdminJobType AdminJobType `json:"admin-job-type"`
Engine SortEngine `json:"sort-engine"`
SortDir string `json:"sort-dir"`
// SortDir is deprecated
SortDir string `json:"-"`

Config *config.ReplicaConfig `json:"config"`
State FeedState `json:"state"`
Expand Down
1 change: 0 additions & 1 deletion cdc/model/changefeed_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -120,7 +120,6 @@ func (s *configSuite) TestFillV1(c *check.C) {
},
StartTs: 417136892416622595,
Engine: "memory",
SortDir: ".",
Config: &config.ReplicaConfig{
CaseSensitive: true,
Filter: &config.FilterConfig{
Expand Down
8 changes: 3 additions & 5 deletions cdc/processor/state_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,7 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) {
"/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225",
},
updateValue: []string{
`{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":".","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"default"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`,
`{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"default"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`,
`{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713,"admin-job-type":0}`,
`{"checkpoint-ts":421980720003809281,"resolved-ts":421980720003809281,"count":0,"error":null}`,
`{"tables":{"45":{"start-ts":421980685886554116,"mark-table-id":0}},"operation":null,"admin-job-type":0}`,
Expand All @@ -100,7 +100,6 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) {
CreateTime: createTime,
StartTs: 421980685886554116,
Engine: model.SortInMemory,
SortDir: ".",
State: "normal",
SyncPointInterval: time.Minute * 10,
Config: &config.ReplicaConfig{
Expand Down Expand Up @@ -141,7 +140,7 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) {
"/tidb/cdc/task/workload/fake-capture-id/test1",
},
updateValue: []string{
`{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":".","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"default"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`,
`{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"default"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`,
`{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713,"admin-job-type":0}`,
`{"checkpoint-ts":421980720003809281,"resolved-ts":421980720003809281,"count":0,"error":null}`,
`{"tables":{"45":{"start-ts":421980685886554116,"mark-table-id":0}},"operation":null,"admin-job-type":0}`,
Expand All @@ -165,7 +164,6 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) {
CreateTime: createTime,
StartTs: 421980685886554116,
Engine: model.SortInMemory,
SortDir: ".",
State: "normal",
SyncPointInterval: time.Minute * 10,
Config: &config.ReplicaConfig{
Expand Down Expand Up @@ -204,7 +202,7 @@ func (s *stateSuite) TestChangefeedStateUpdate(c *check.C) {
"/tidb/cdc/capture/6bbc01c8-0605-4f86-a0f9-b3119109b225",
},
updateValue: []string{
`{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","sort-dir":".","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"default"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`,
`{"sink-uri":"blackhole://","opts":{},"create-time":"2020-02-02T00:00:00.000000+00:00","start-ts":421980685886554116,"target-ts":0,"admin-job-type":0,"sort-engine":"memory","config":{"case-sensitive":true,"enable-old-value":false,"force-replicate":false,"check-gc-safe-point":true,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"ddl-allow-list":null},"mounter":{"worker-num":16},"sink":{"dispatchers":null,"protocol":"default"},"cyclic-replication":{"enable":false,"replica-id":0,"filter-replica-ids":null,"id-buckets":0,"sync-ddl":false},"scheduler":{"type":"table-number","polling-time":-1}},"state":"normal","history":null,"error":null,"sync-point-enabled":false,"sync-point-interval":600000000000}`,
`{"resolved-ts":421980720003809281,"checkpoint-ts":421980719742451713,"admin-job-type":0}`,
`{"checkpoint-ts":421980720003809281,"resolved-ts":421980720003809281,"count":0,"error":null}`,
`{"tables":{"45":{"start-ts":421980685886554116,"mark-table-id":0}},"operation":null,"admin-job-type":0}`,
Expand Down
20 changes: 6 additions & 14 deletions cmd/client_changefeed.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -341,29 +341,22 @@ func verifyChangefeedParamers(ctx context.Context, cmd *cobra.Command, isCreate
TargetTs: targetTs,
Config: cfg,
Engine: sortEngine,
SortDir: sortDir,
State: model.StateNormal,
SyncPointEnabled: syncPointEnabled,
SyncPointInterval: syncPointInterval,
CreatorVersion: version.ReleaseVersion,
}

if info.SortDir != "" {
cmd.Printf("[WARN] --sort-dir is deprecated in changefeed settings. "+
"Please use `cdc server --sort-dir` if possible. "+
"If you wish to continue, make sure %s is writable ON EACH SERVER where you run TiCDC", info.SortDir)
}

if info.Engine != model.SortInMemory && (info.SortDir == ".") {
cmd.Printf("[WARN] you are using the directory containing the cdc binary as sort-dir. " +
"make sure that is what you intend, and that the directory is writable. " +
"Adjust \"sort-dir\" accordingly if you'd like to use another directory. ")
if sortDir != "" {
cmd.Printf("[WARN] --sort-dir is deprecated in changefeed settings. " +
"Please use `cdc server --sort-dir` if possible. " +
"The sort-dir here will be no-op\n")
}

if info.Engine == model.SortInFile {
cmd.Printf("[WARN] file sorter is deprecated. " +
"make sure that you DO NOT use it in production. " +
"Adjust \"sort-engine\" to make use of the right sorter.")
"Adjust \"sort-engine\" to make use of the right sorter.\n")
}

tz, err := util.GetTimezone(timezone)
Expand Down Expand Up @@ -440,6 +433,7 @@ func changefeedConfigVariables(command *cobra.Command) {
command.PersistentFlags().BoolVar(&cyclicSyncDDL, "cyclic-sync-ddl", true, "(Expremental) Cyclic replication sync DDL of changefeed")
command.PersistentFlags().BoolVar(&syncPointEnabled, "sync-point", false, "(Expremental) Set and Record syncpoint in replication(default off)")
command.PersistentFlags().DurationVar(&syncPointInterval, "sync-interval", 10*time.Minute, "(Expremental) Set the interval for syncpoint in replication(default 10min)")
command.PersistentFlags().MarkHidden("sort-dir") //nolint:errcheck
}

func newCreateChangefeedCommand() *cobra.Command {
Expand Down Expand Up @@ -529,8 +523,6 @@ func newUpdateChangefeedCommand() *cobra.Command {

case "sort-engine":
info.Engine = sortEngine
case "sort-dir":
info.SortDir = sortDir
case "cyclic-replica-id":
filter := make([]uint64, 0, len(cyclicFilterReplicaIDs))
for _, id := range cyclicFilterReplicaIDs {
Expand Down
2 changes: 1 addition & 1 deletion tests/unified_sorter/run.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,7 @@ function run() {
esac
sort_dir="$WORK_DIR/unified_sort_cache"
mkdir $sort_dir
run_cdc_cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" --sort-engine="unified" --sort-dir="$sort_dir"
run_cdc_cli changefeed create --start-ts=$start_ts --sink-uri="$SINK_URI" --sort-engine="unified"
if [ "$SINK_TYPE" == "kafka" ]; then
run_kafka_consumer $WORK_DIR "kafka://127.0.0.1:9092/$TOPIC_NAME?partition-num=4&version=${KAFKA_VERSION}"
fi
Expand Down