Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

cdc: Add storage sink pipeline #1985

Merged
merged 1 commit into from
Apr 19, 2023
Merged

cdc: Add storage sink pipeline #1985

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 30 additions & 0 deletions pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
apiVersion: v1
kind: Pod
spec:
securityContext:
fsGroup: 1000
containers:
- name: golang
image: "hub.pingcap.net/jenkins/centos7_golang-1.20:latest"
tty: true
resources:
requests:
memory: 12Gi
cpu: "4"
limits:
memory: 16Gi
cpu: "6"
- name: net-tool
image: wbitt/network-multitool
tty: true
resources:
limits:
memory: 128Mi
cpu: 100m
- name: report
image: hub.pingcap.net/jenkins/python3-requests:latest
tty: true
resources:
limits:
memory: 256Mi
cpu: 100m
185 changes: 185 additions & 0 deletions pipelines/pingcap/tiflow/latest/pull_cdc_integration_storage_test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,185 @@
// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline
// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984
// should triggerd for master branches
@Library('tipipeline') _

final K8S_NAMESPACE = "jenkins-tiflow"
final GIT_FULL_REPO_NAME = 'pingcap/tiflow'
final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
final GIT_CREDENTIALS_ID2 = 'github-pr-diff-token'
final POD_TEMPLATE_FILE = 'pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml'
final REFS = readJSON(text: params.JOB_SPEC).refs
def skipRemainingStages = false

pipeline {
agent {
kubernetes {
namespace K8S_NAMESPACE
yamlFile POD_TEMPLATE_FILE
defaultContainer 'golang'
}
}
environment {
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
}
options {
timeout(time: 60, unit: 'MINUTES')
parallelsAlwaysFailFast()
}
stages {
stage('Debug info') {
steps {
sh label: 'Debug info', script: """
printenv
echo "-------------------------"
go env
echo "-------------------------"
echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash"
"""
container(name: 'net-tool') {
sh 'dig github.com'
}
}
}
stage('Check diff files') {
steps {
container("golang") {
script {
def pr_diff_files = component.getPrDiffFiles(GIT_FULL_REPO_NAME, REFS.pulls[0].number, GIT_CREDENTIALS_ID2)
def pattern = /(^dm\/|^engine\/).*$/
println "pr_diff_files: ${pr_diff_files}"
// if all diff files start with dm/, skip cdc integration test
def matched = component.patternMatchAllFiles(pattern, pr_diff_files)
if (matched) {
println "matched, all diff files full path start with dm/ or engine/, current pr is dm/engine's pr(not related to ticdc), skip cdc integration test"
currentBuild.result = 'SUCCESS'
skipRemainingStages = true
return
}
}
}
}
}
stage('Checkout') {
when { expression { !skipRemainingStages} }
options { timeout(time: 10, unit: 'MINUTES') }
steps {
dir("tiflow") {
cache(path: "./", filter: '**/*', key: "git/pingcap/tiflow/rev-${REFS.pulls[0].sha}", restoreKeys: ['git/pingcap/tiflow/rev-']) {
retry(2) {
checkout(
changelog: false,
poll: false,
scm: [
$class: 'GitSCM', branches: [[name: REFS.pulls[0].sha ]],
doGenerateSubmoduleConfigurations: false,
extensions: [
[$class: 'PruneStaleBranch'],
[$class: 'CleanBeforeCheckout'],
[$class: 'CloneOption', timeout: 15],
],
submoduleCfg: [],
userRemoteConfigs: [[
refspec: "+refs/pull/${REFS.pulls[0].number}/*:refs/remotes/origin/pr/${REFS.pulls[0].number}/*",
url: "https://github.com/${GIT_FULL_REPO_NAME}.git",
]],
]
)
}
}
}
}
}
stage("prepare") {
when { expression { !skipRemainingStages} }
options { timeout(time: 20, unit: 'MINUTES') }
steps {
dir("third_party_download") {
retry(2) {
sh label: "download third_party", script: """
cd ../tiflow && ./scripts/download-integration-test-binaries.sh ${REFS.base_ref} && ls -alh ./bin
make check_third_party_binary
cd - && mkdir -p bin && mv ../tiflow/bin/* ./bin/
ls -alh ./bin
./bin/tidb-server -V
./bin/pd-server -V
./bin/tikv-server -V
./bin/tiflash --version
./bin/sync_diff_inspector --version
"""
}
}
dir("tiflow") {
cache(path: "./bin", filter: '**/*', key: "git/pingcap/tiflow/cdc-integration-test-binarys-${REFS.pulls[0].sha}") {
// build cdc, kafka_consumer, storage_consumer, cdc.test for integration test
// only build binarys if not exist, use the cached binarys if exist
sh label: "prepare", script: """
ls -alh ./bin
[ -f ./bin/cdc ] || make cdc
[ -f ./bin/cdc_kafka_consumer ] || make kafka_consumer
[ -f ./bin/cdc_storage_consumer ] || make storage_consumer
[ -f ./bin/cdc.test ] || make integration_test_build
ls -alh ./bin
./bin/cdc version
"""
}
cache(path: "./", filter: '**/*', key: "ws/${BUILD_TAG}/tiflow-cdc") {
sh label: "prepare", script: """
cp -r ../third_party_download/bin/* ./bin/
ls -alh ./bin
"""
}
}
}
}

stage('Tests') {
when { expression { !skipRemainingStages} }
matrix {
axes {
axis {
name 'TEST_GROUP'
values 'G00', 'G01', 'G02'
}
}
agent{
kubernetes {
namespace K8S_NAMESPACE
yamlFile POD_TEMPLATE_FILE
defaultContainer 'golang'
}
}
stages {
stage("Test") {
options { timeout(time: 40, unit: 'MINUTES') }
environment {
TICDC_CODECOV_TOKEN = credentials('codecov-token-tiflow')
TICDC_COVERALLS_TOKEN = credentials('coveralls-token-tiflow')
}
steps {
dir('tiflow') {
cache(path: "./", filter: '**/*', key: "ws/${BUILD_TAG}/tiflow-cdc") {
sh label: "${TEST_GROUP}", script: """
rm -rf /tmp/tidb_cdc_test && mkdir -p /tmp/tidb_cdc_test
chmod +x ./tests/integration_tests/run_group.sh
./tests/integration_tests/run_group.sh storage ${TEST_GROUP}
"""
}
}
}
post {
failure {
sh label: "collect logs", script: """
ls /tmp/tidb_cdc_test/
tar -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/tidb_cdc_test/ -type f -name "*.log")
ls -alh log-${TEST_GROUP}.tar.gz
"""
archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true
}
}
}
}
}
}
}
}