Skip to content

Commit

Permalink
Merge branch 'release-5.3' into cherry-pick-3525-to-release-5.3
Browse files Browse the repository at this point in the history
  • Loading branch information
ben1009 authored Dec 28, 2021
2 parents 4b86b67 + 1eca2a5 commit 37055ba
Show file tree
Hide file tree
Showing 851 changed files with 6,819 additions and 3,302 deletions.
12 changes: 12 additions & 0 deletions .golangci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ linters:
- unconvert
- unparam
- revive
- depguard

linters-settings:
revive:
Expand All @@ -28,3 +29,14 @@ linters-settings:
- name: superfluous-else
- name: modifies-parameter
- name: unreachable-code

depguard:
list-type: blacklist
include-go-root: false
packages:
- log
- github.com/juju/errors
packages-with-error-message:
# specify an error message to output when a blacklisted package is used
- log: "logging is allowed only by pingcap/log"
- github.com/juju/errors: "error handling is allowed only by pingcap/errors"
4 changes: 2 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
FROM golang:1.16-alpine as builder
RUN apk add --no-cache git make bash
WORKDIR /go/src/github.com/pingcap/ticdc
WORKDIR /go/src/github.com/pingcap/tiflow
COPY . .
ENV CDC_ENABLE_VENDOR=0
RUN make

FROM alpine:3.12
RUN apk add --no-cache tzdata bash curl socat
COPY --from=builder /go/src/github.com/pingcap/ticdc/bin/cdc /cdc
COPY --from=builder /go/src/github.com/pingcap/tiflow/bin/cdc /cdc
EXPOSE 8300
CMD [ "/cdc" ]
4 changes: 2 additions & 2 deletions Dockerfile.development
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
FROM golang:1.16-alpine3.12 as builder
RUN apk add --no-cache git make bash
WORKDIR /go/src/github.com/pingcap/ticdc
WORKDIR /go/src/github.com/pingcap/tiflow
COPY . .
ENV CDC_ENABLE_VENDOR=1
RUN go mod vendor
Expand All @@ -10,6 +10,6 @@ RUN make failpoint-disable

FROM alpine:3.12
RUN apk add --no-cache tzdata bash curl socat
COPY --from=builder /go/src/github.com/pingcap/ticdc/bin/cdc /cdc
COPY --from=builder /go/src/github.com/pingcap/tiflow/bin/cdc /cdc
EXPOSE 8300
CMD [ "/cdc" ]
30 changes: 15 additions & 15 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
integration_test_build integration_test integration_test_mysql integration_test_kafka bank \
dm dm-master dm-worker dmctl dm-portal dm-syncer dm_coverage

PROJECT=ticdc
PROJECT=tiflow
P=3

FAIL_ON_STDOUT := awk '{ print } END { if (NR > 0) { exit 1 } }'
Expand All @@ -30,11 +30,11 @@ GOTESTNORACE := CGO_ENABLED=1 $(GO) test -p $(P)
ARCH := "$(shell uname -s)"
LINUX := "Linux"
MAC := "Darwin"
CDC_PKG := github.com/pingcap/ticdc
DM_PKG := github.com/pingcap/ticdc/dm
CDC_PKG := github.com/pingcap/tiflow
DM_PKG := github.com/pingcap/tiflow/dm
PACKAGE_LIST := go list ./... | grep -vE 'vendor|proto|ticdc\/tests|integration|testing_utils|pb|pbmock'
PACKAGE_LIST_WITHOUT_DM := $(PACKAGE_LIST) | grep -vE 'github.com/pingcap/ticdc/dm'
DM_PACKAGE_LIST := go list github.com/pingcap/ticdc/dm/... | grep -vE 'pb|pbmock|dm/cmd'
PACKAGE_LIST_WITHOUT_DM := $(PACKAGE_LIST) | grep -vE 'github.com/pingcap/tiflow/dm'
DM_PACKAGE_LIST := go list github.com/pingcap/tiflow/dm/... | grep -vE 'pb|pbmock|dm/cmd'
PACKAGES := $$($(PACKAGE_LIST))
PACKAGES_WITHOUT_DM := $$($(PACKAGE_LIST_WITHOUT_DM))
DM_PACKAGES := $$($(DM_PACKAGE_LIST))
Expand Down Expand Up @@ -141,8 +141,8 @@ check_third_party_binary:
integration_test_build: check_failpoint_ctl
$(FAILPOINT_ENABLE)
$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
-coverpkg=github.com/pingcap/ticdc/... \
-o bin/cdc.test github.com/pingcap/ticdc/cmd/cdc \
-coverpkg=github.com/pingcap/tiflow/... \
-o bin/cdc.test github.com/pingcap/tiflow/cmd/cdc \
|| { $(FAILPOINT_DISABLE); exit 1; }
$(GOBUILD) -ldflags '$(LDFLAGS)' -o bin/cdc ./cmd/cdc/main.go \
|| { $(FAILPOINT_DISABLE); exit 1; }
Expand Down Expand Up @@ -273,20 +273,20 @@ dm_unit_test: check_failpoint_ctl
dm_integration_test_build: check_failpoint_ctl
$(FAILPOINT_ENABLE)
$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
-coverpkg=github.com/pingcap/ticdc/dm/... \
-o bin/dm-worker.test github.com/pingcap/ticdc/dm/cmd/dm-worker \
-coverpkg=github.com/pingcap/tiflow/dm/... \
-o bin/dm-worker.test github.com/pingcap/tiflow/dm/cmd/dm-worker \
|| { $(FAILPOINT_DISABLE); exit 1; }
$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
-coverpkg=github.com/pingcap/ticdc/dm/... \
-o bin/dm-master.test github.com/pingcap/ticdc/dm/cmd/dm-master \
-coverpkg=github.com/pingcap/tiflow/dm/... \
-o bin/dm-master.test github.com/pingcap/tiflow/dm/cmd/dm-master \
|| { $(FAILPOINT_DISABLE); exit 1; }
$(GOTESTNORACE) -ldflags '$(LDFLAGS)' -c -cover -covermode=count \
-coverpkg=github.com/pingcap/ticdc/dm/... \
-o bin/dmctl.test github.com/pingcap/ticdc/dm/cmd/dm-ctl \
-coverpkg=github.com/pingcap/tiflow/dm/... \
-o bin/dmctl.test github.com/pingcap/tiflow/dm/cmd/dm-ctl \
|| { $(FAILPOINT_DISABLE); exit 1; }
$(GOTEST) -ldflags '$(LDFLAGS)' -c -cover -covermode=atomic \
-coverpkg=github.com/pingcap/ticdc/dm/... \
-o bin/dm-syncer.test github.com/pingcap/ticdc/dm/cmd/dm-syncer \
-coverpkg=github.com/pingcap/tiflow/dm/... \
-o bin/dm-syncer.test github.com/pingcap/tiflow/dm/cmd/dm-syncer \
|| { $(FAILPOINT_DISABLE); exit 1; }
$(FAILPOINT_DISABLE)
./dm/tests/prepare_tools.sh
Expand Down
10 changes: 5 additions & 5 deletions README.md
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
# TiCDC

[![Build Status](https://github.com/pingcap/ticdc/actions/workflows/check_and_build.yaml/badge.svg?branch=master)](https://github.com/pingcap/ticdc/actions/workflows/check_and_build.yaml?query=event%3Apush+branch%3Amaster)
[![codecov](https://codecov.io/gh/pingcap/ticdc/branch/master/graph/badge.svg)](https://codecov.io/gh/pingcap/ticdc)
[![Coverage Status](https://coveralls.io/repos/github/pingcap/ticdc/badge.svg)](https://coveralls.io/github/pingcap/ticdc)
[![LICENSE](https://img.shields.io/github/license/pingcap/ticdc.svg)](https://github.com/pingcap/ticdc/blob/master/LICENSE)
[![Go Report Card](https://goreportcard.com/badge/github.com/pingcap/ticdc)](https://goreportcard.com/report/github.com/pingcap/ticdc)
[![Build Status](https://github.com/pingcap/tiflow/actions/workflows/check_and_build.yaml/badge.svg?branch=master)](https://github.com/pingcap/tiflow/actions/workflows/check_and_build.yaml?query=event%3Apush+branch%3Amaster)
[![codecov](https://codecov.io/gh/pingcap/tiflow/branch/master/graph/badge.svg)](https://codecov.io/gh/pingcap/tiflow)
[![Coverage Status](https://coveralls.io/repos/github/pingcap/tiflow/badge.svg)](https://coveralls.io/github/pingcap/tiflow)
[![LICENSE](https://img.shields.io/github/license/pingcap/tiflow.svg)](https://github.com/pingcap/tiflow/blob/master/LICENSE)
[![Go Report Card](https://goreportcard.com/badge/github.com/pingcap/tiflow)](https://goreportcard.com/report/github.com/pingcap/tiflow)

**TiCDC** is [TiDB](https://docs.pingcap.com/tidb/stable)'s change data capture framework. It supports replicating change data to various downstreams, including MySQL protocol-compatible databases, message queues via the open CDC protocol and other systems such as local file storage.

Expand Down
31 changes: 19 additions & 12 deletions cdc/capture/capture.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,18 @@ import (
"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
"github.com/pingcap/log"
"github.com/pingcap/ticdc/cdc/kv"
"github.com/pingcap/ticdc/cdc/model"
"github.com/pingcap/ticdc/cdc/owner"
"github.com/pingcap/ticdc/cdc/processor"
"github.com/pingcap/ticdc/pkg/config"
cdcContext "github.com/pingcap/ticdc/pkg/context"
cerror "github.com/pingcap/ticdc/pkg/errors"
"github.com/pingcap/ticdc/pkg/etcd"
"github.com/pingcap/ticdc/pkg/orchestrator"
"github.com/pingcap/ticdc/pkg/pdtime"
"github.com/pingcap/ticdc/pkg/version"
tidbkv "github.com/pingcap/tidb/kv"
"github.com/pingcap/tiflow/cdc/kv"
"github.com/pingcap/tiflow/cdc/model"
"github.com/pingcap/tiflow/cdc/owner"
"github.com/pingcap/tiflow/cdc/processor"
"github.com/pingcap/tiflow/pkg/config"
cdcContext "github.com/pingcap/tiflow/pkg/context"
cerror "github.com/pingcap/tiflow/pkg/errors"
"github.com/pingcap/tiflow/pkg/etcd"
"github.com/pingcap/tiflow/pkg/orchestrator"
"github.com/pingcap/tiflow/pkg/pdtime"
"github.com/pingcap/tiflow/pkg/version"
pd "github.com/tikv/pd/client"
"go.etcd.io/etcd/clientv3/concurrency"
"go.etcd.io/etcd/mvcc"
Expand Down Expand Up @@ -81,6 +81,12 @@ func NewCapture(pdClient pd.Client, kvStorage tidbkv.Storage, etcdClient *etcd.C
}
}

func NewCapture4Test() *Capture {
return &Capture{
info: &model.CaptureInfo{ID: "capture-for-test", AdvertiseAddr: "127.0.0.1", Version: "test"},
}
}

func (c *Capture) reset(ctx context.Context) error {
c.captureMu.Lock()
defer c.captureMu.Unlock()
Expand Down Expand Up @@ -283,7 +289,8 @@ func (c *Capture) runEtcdWorker(ctx cdcContext.Context, reactor orchestrator.Rea
if err != nil {
return errors.Trace(err)
}
if err := etcdWorker.Run(ctx, c.session, timerInterval); err != nil {
captureAddr := c.info.AdvertiseAddr
if err := etcdWorker.Run(ctx, c.session, timerInterval, captureAddr); err != nil {
// We check ttl of lease instead of check `session.Done`, because
// `session.Done` is only notified when etcd client establish a
// new keepalive request, there could be a time window as long as
Expand Down
51 changes: 51 additions & 0 deletions cdc/capture/http_errors.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
// Copyright 2021 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package capture

import (
"strings"

"github.com/pingcap/errors"
cerror "github.com/pingcap/tiflow/pkg/errors"
)

// httpBadRequestError is some errors that will cause a BadRequestError in http handler
var httpBadRequestError = []*errors.Error{
cerror.ErrAPIInvalidParam, cerror.ErrSinkURIInvalid, cerror.ErrStartTsBeforeGC,
cerror.ErrChangeFeedNotExists, cerror.ErrTargetTsBeforeStartTs, cerror.ErrTableIneligible,
cerror.ErrFilterRuleInvalid, cerror.ErrChangefeedUpdateRefused, cerror.ErrMySQLConnectionError,
cerror.ErrMySQLInvalidConfig,
}

// IsHTTPBadRequestError check if a error is a http bad request error
func IsHTTPBadRequestError(err error) bool {
if err == nil {
return false
}
for _, e := range httpBadRequestError {
if e.Equal(err) {
return true
}

rfcCode, ok := cerror.RFCCode(err)
if ok && e.RFCCode() == rfcCode {
return true
}

if strings.Contains(err.Error(), string(e.RFCCode())) {
return true
}
}
return false
}
33 changes: 33 additions & 0 deletions cdc/capture/http_errors_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
// Copyright 2021 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package capture

import (
"testing"

"github.com/pingcap/errors"
cerror "github.com/pingcap/tiflow/pkg/errors"
"github.com/stretchr/testify/require"
)

func TestIsHTTPBadRequestError(t *testing.T) {
err := cerror.ErrAPIInvalidParam.GenWithStack("aa")
require.Equal(t, true, IsHTTPBadRequestError(err))
err = cerror.ErrAPIInvalidParam.Wrap(errors.New("aa"))
require.Equal(t, true, IsHTTPBadRequestError(err))
err = cerror.ErrPDEtcdAPIError.GenWithStack("aa")
require.Equal(t, false, IsHTTPBadRequestError(err))
err = nil
require.Equal(t, false, IsHTTPBadRequestError(err))
}
Loading

0 comments on commit 37055ba

Please sign in to comment.