Skip to content

Commit

Permalink
add storage sink pipieline
Browse files Browse the repository at this point in the history
  • Loading branch information
@CharlesCheung96
CharlesCheung96 committed Apr 19, 2023
1 parent f793f84 commit 0d4dc55
Show file tree
Hide file tree
Showing 2 changed files with 216 additions and 0 deletions.
30 changes: 30 additions & 0 deletions pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,30 @@
apiVersion: v1
kind: Pod
spec:
securityContext:
fsGroup: 1000
containers:
- name: golang
image: "hub.pingcap.net/jenkins/centos7_golang-1.20:latest"
tty: true
resources:
requests:
memory: 12Gi
cpu: "4"
limits:
memory: 16Gi
cpu: "6"
- name: net-tool
image: wbitt/network-multitool
tty: true
resources:
limits:
memory: 128Mi
cpu: 100m
- name: report
image: hub.pingcap.net/jenkins/python3-requests:latest
tty: true
resources:
limits:
memory: 256Mi
cpu: 100m
186 changes: 186 additions & 0 deletions pipelines/pingcap/tiflow/latest/pull_cdc_integration_storage_test.groovy
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
// REF: https://www.jenkins.io/doc/book/pipeline/syntax/#declarative-pipeline
// Keep small than 400 lines: https://issues.jenkins.io/browse/JENKINS-37984
// should triggerd for master branches
@Library('tipipeline') _

final K8S_NAMESPACE = "jenkins-tiflow"
final GIT_FULL_REPO_NAME = 'pingcap/tiflow'
final GIT_CREDENTIALS_ID = 'github-sre-bot-ssh'
final GIT_CREDENTIALS_ID2 = 'github-pr-diff-token'
final POD_TEMPLATE_FILE = 'pipelines/pingcap/tiflow/latest/pod-pull_cdc_integration_storage_test.yaml'
final REFS = readJSON(text: params.JOB_SPEC).refs
def skipRemainingStages = false

pipeline {
agent {
kubernetes {
namespace K8S_NAMESPACE
yamlFile POD_TEMPLATE_FILE
defaultContainer 'golang'
}
}
environment {
FILE_SERVER_URL = 'http://fileserver.pingcap.net'
}
options {
timeout(time: 60, unit: 'MINUTES')
parallelsAlwaysFailFast()
}
stages {
stage('Debug info') {
steps {
sh label: 'Debug info', script: """
printenv
echo "-------------------------"
go env
echo "-------------------------"
echo "debug command: kubectl -n ${K8S_NAMESPACE} exec -ti ${NODE_NAME} bash"
"""
container(name: 'net-tool') {
sh 'dig github.com'
}
}
}
stage('Check diff files') {
steps {
container("golang") {
script {
def pr_diff_files = component.getPrDiffFiles(GIT_FULL_REPO_NAME, REFS.pulls[0].number, GIT_CREDENTIALS_ID2)
def pattern = /(^dm\/|^engine\/).*$/
println "pr_diff_files: ${pr_diff_files}"
// if all diff files start with dm/, skip cdc integration test
def matched = component.patternMatchAllFiles(pattern, pr_diff_files)
if (matched) {
println "matched, all diff files full path start with dm/ or engine/, current pr is dm/engine's pr(not related to ticdc), skip cdc integration test"
currentBuild.result = 'SUCCESS'
skipRemainingStages = true
return
}
}
}
}
}
stage('Checkout') {
when { expression { !skipRemainingStages} }
options { timeout(time: 10, unit: 'MINUTES') }
steps {
dir("tiflow") {
cache(path: "./", filter: '**/*', key: "git/pingcap/tiflow/rev-${REFS.pulls[0].sha}", restoreKeys: ['git/pingcap/tiflow/rev-']) {
retry(2) {
checkout(
changelog: false,
poll: false,
scm: [
$class: 'GitSCM', branches: [[name: REFS.pulls[0].sha ]],
doGenerateSubmoduleConfigurations: false,
extensions: [
[$class: 'PruneStaleBranch'],
[$class: 'CleanBeforeCheckout'],
[$class: 'CloneOption', timeout: 15],
],
submoduleCfg: [],
userRemoteConfigs: [[
refspec: "+refs/pull/${REFS.pulls[0].number}/*:refs/remotes/origin/pr/${REFS.pulls[0].number}/*",
url: "https://github.com/${GIT_FULL_REPO_NAME}.git",
]],
]
)
}
}
}
}
}
stage("prepare") {
when { expression { !skipRemainingStages} }
options { timeout(time: 20, unit: 'MINUTES') }
steps {
dir("third_party_download") {
retry(2) {
sh label: "download third_party", script: """
cd ../tiflow && ./scripts/download-integration-test-binaries.sh ${REFS.base_ref} && ls -alh ./bin
make check_third_party_binary
cd - && mkdir -p bin && mv ../tiflow/bin/* ./bin/
ls -alh ./bin
./bin/tidb-server -V
./bin/pd-server -V
./bin/tikv-server -V
./bin/tiflash --version
./bin/sync_diff_inspector --version
"""
}
}
dir("tiflow") {
cache(path: "./bin", filter: '**/*', key: "git/pingcap/tiflow/cdc-integration-test-binarys-${REFS.pulls[0].sha}") {
// build cdc, kafka_consumer, storage_consumer, cdc.test for integration test
// only build binarys if not exist, use the cached binarys if exist
sh label: "prepare", script: """
ls -alh ./bin
[ -f ./bin/cdc ] || make cdc
[ -f ./bin/cdc_kafka_consumer ] || make kafka_consumer
[ -f ./bin/cdc_storage_consumer ] || make storage_consumer
[ -f ./bin/cdc.test ] || make integration_test_build
ls -alh ./bin
./bin/cdc version
"""
}
cache(path: "./", filter: '**/*', key: "ws/${BUILD_TAG}/tiflow-cdc") {
sh label: "prepare", script: """
cp -r ../third_party_download/bin/* ./bin/
ls -alh ./bin
"""
}
}
}
}

stage('Tests') {
when { expression { !skipRemainingStages} }
matrix {
axes {
axis {
name 'TEST_GROUP'
values 'G00', 'G01', 'G02', 'G03', 'G04', 'G05', 'G06', 'G07', 'G08', 'G09', 'G10', 'G11', 'G12', 'G13',
'G14', 'G15', 'G16', 'G17', 'G18', 'G19', 'G20', 'G21', 'G22', 'G23', 'G24'
}
}
agent{
kubernetes {
namespace K8S_NAMESPACE
yamlFile POD_TEMPLATE_FILE
defaultContainer 'golang'
}
}
stages {
stage("Test") {
options { timeout(time: 40, unit: 'MINUTES') }
environment {
TICDC_CODECOV_TOKEN = credentials('codecov-token-tiflow')
TICDC_COVERALLS_TOKEN = credentials('coveralls-token-tiflow')
}
steps {
dir('tiflow') {
cache(path: "./", filter: '**/*', key: "ws/${BUILD_TAG}/tiflow-cdc") {
sh label: "${TEST_GROUP}", script: """
rm -rf /tmp/tidb_cdc_test && mkdir -p /tmp/tidb_cdc_test
chmod +x ./tests/integration_tests/run_group.sh
./tests/integration_tests/run_group.sh storage ${TEST_GROUP}
"""
}
}
}
post {
failure {
sh label: "collect logs", script: """
ls /tmp/tidb_cdc_test/
tar -cvzf log-${TEST_GROUP}.tar.gz \$(find /tmp/tidb_cdc_test/ -type f -name "*.log")
ls -alh log-${TEST_GROUP}.tar.gz
"""
archiveArtifacts artifacts: "log-${TEST_GROUP}.tar.gz", fingerprint: true
}
}
}
}
}
}
}
}

0 comments on commit 0d4dc55

Please sign in to comment.