Skip to content
New issue

Have a question about this project? # for a free GitHub account to open an issue and contact its maintainers and the community.

#

By clicking “#”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? # to your account

chore: update dm ci #346

Merged
merged 3 commits into from
Aug 16, 2021
Merged

chore: update dm ci #346

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
b2f1824
split dm others test
Ehco1996 Jul 30, 2021
13ba037
add run_tls_source_it_test
Ehco1996 Aug 1, 2021
e8cd8eb
revert to mysql8
Ehco1996 Aug 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
129 changes: 121 additions & 8 deletions jenkins/pipelines/ci/dm/dm_ghpr_new_test.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@
if ("${ghprbTargetBranch}" == 'release-1.0') {
MYSQL_ARGS = '--ssl=OFF --log-bin --binlog-format=ROW --enforce-gtid-consistency=ON --gtid-mode=ON --server-id=1 --default-authentication-plugin=mysql_native_password'
}else {
MYSQL_ARGS = '--log-bin --binlog-format=ROW --enforce-gtid-consistency=ON --gtid-mode=ON --server-id=1 --default-authentication-plugin=mysql_native_password'
MYSQL_ARGS = '--ssl=ON --log-bin --binlog-format=ROW --enforce-gtid-consistency=ON --gtid-mode=ON --server-id=1 --default-authentication-plugin=mysql_native_password'
}

MYSQL_HOST = '127.0.0.1'
Expand All @@ -44,7 +44,7 @@ def boolean isBranchMatched(List<String> branches, String targetBranch) {
return false
}

def isNeedGo1160 = isBranchMatched(['master', "release-2.0"], ghprbTargetBranch)
def isNeedGo1160 = isBranchMatched(['master', 'release-2.0'], ghprbTargetBranch)
if (isNeedGo1160) {
println 'This build use go1.16'
GO_BUILD_SLAVE = GO1160_BUILD_SLAVE
Expand Down Expand Up @@ -110,6 +110,8 @@ def build_dm_bin() {
unstash 'dm'
ws = pwd()
dir('go/src/github.com/pingcap/dm') {
println "debug command:\nkubectl -n jenkins-tidb exec -ti ${env.NODE_NAME} bash"

// build it test bin
sh 'make dm_integration_test_build'

Expand All @@ -130,8 +132,6 @@ def build_dm_bin() {
// use a new version of gh-ost to overwrite the one in container("golang") (1.0.47 --> 1.1.0)
sh 'curl -L https://github.com/github/gh-ost/releases/download/v1.1.0/gh-ost-binary-linux-20200828140552.tar.gz | tar xz'
sh 'mv gh-ost bin/'

println "debug command:\nkubectl -n jenkins-tidb exec -ti ${env.NODE_NAME} bash"
}
dir("${ws}") {
stash includes: 'go/src/github.com/pingcap/dm/**', name: 'dm-with-bin', useDefaultExcludes: false
Expand Down Expand Up @@ -193,6 +193,110 @@ def run_single_unit_test(String case_name) {
}
}


def run_tls_source_it_test(String case_name) {
def label = 'dm-integration-test'
podTemplate(label: label,
nodeSelector: 'role_type=slave',
namespace: 'jenkins-tidb',
containers: [
containerTemplate(
name: 'golang', alwaysPullImage: false,
image: "${POD_GO_DOCKER_IMAGE}", ttyEnabled: true,
resourceRequestCpu: '2000m', resourceRequestMemory: '4Gi',
command: 'cat'),
containerTemplate(
name: 'mysql1', alwaysPullImage: false,
image: 'hub.pingcap.net/jenkins/mysql:5.7',ttyEnabled: true,
resourceRequestCpu: '1000m', resourceRequestMemory: '1Gi',
envVars: [
envVar(key: 'MYSQL_ROOT_PASSWORD', value: "${MYSQL_PSWD}"),
],
args: "${MYSQL_ARGS}"),
// mysql 8
containerTemplate(
name: 'mysql2', alwaysPullImage: false,
image: 'hub.pingcap.net/zhangxuecheng/mysql:8.0.21',ttyEnabled: true,
resourceRequestCpu: '1000m', resourceRequestMemory: '1Gi',
envVars: [
envVar(key: 'MYSQL_ROOT_PASSWORD', value: "${MYSQL_PSWD}"),
envVar(key: 'MYSQL_TCP_PORT', value: "${MYSQL2_PORT}")
],
args: "${MYSQL_ARGS}")
]
) {
node(label) {
println "${NODE_NAME}"
println "debug command: \nkubectl -n jenkins-tidb exec -ti ${env.NODE_NAME} -c golang bash"
// stash ssl certs to jenkins, i don't know why if filename == client-key.pem , the stash will fail
// so just hack the filename
container('mysql1') {
def ws = pwd()
deleteDir()
sh "set +e && for i in {1..90}; do mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent; if [ \$? -eq 0 ]; then set -e; break; else if [ \$i -eq 90 ]; then set -e; exit 2; fi; sleep 2; fi; done"
sh "cp -r /var/lib/mysql/*.pem ."
sh "ls"
sh "pwd"
sh "cat client-key.pem > client.key.pem"
sh "cat client.key.pem"
stash includes: 'ca.pem,client-cert.pem,client.key.pem', name: "mysql-certs", useDefaultExcludes: false
}

container('golang') {
def ws = pwd()
deleteDir()
unstash(name: 'mysql-certs')
sh "ls"
sh "mv client.key.pem client-key.pem"
sh "sudo mkdir -p /var/lib/mysql"
sh "sudo chmod 777 /var/lib/mysql"
sh "cp *.pem /var/lib/mysql/"
sh "ls /var/lib/mysql"

unstash 'dm-with-bin'
dir('go/src/github.com/pingcap/dm') {
try {
sh"""
# use a new version of gh-ost to overwrite the one in container("golang") (1.0.47 --> 1.1.0)
export PATH=bin:$PATH
rm -rf /tmp/dm_test
mkdir -p /tmp/dm_test
export MYSQL_HOST1=${MYSQL_HOST}
export MYSQL_PORT1=${MYSQL_PORT}
export MYSQL_HOST2=${MYSQL_HOST}
export MYSQL_PORT2=${MYSQL2_PORT}
# wait for mysql container ready.
set +e && for i in {1..90}; do mysqladmin ping -h127.0.0.1 -P 3306 -p123456 -uroot --silent; if [ \$? -eq 0 ]; then set -e; break; else if [ \$i -eq 90 ]; then set -e; exit 2; fi; sleep 2; fi; done
set +e && for i in {1..90}; do mysqladmin ping -h127.0.0.1 -P 3307 -p123456 -uroot --silent; if [ \$? -eq 0 ]; then set -e; break; else if [ \$i -eq 90 ]; then set -e; exit 2; fi; sleep 2; fi; done
# run test
export GOPATH=\$GOPATH:${ws}/go
make integration_test CASE="${case_name}"
# upload coverage
rm -rf cov_dir
mkdir -p cov_dir
ls /tmp/dm_test
cp /tmp/dm_test/cov*out cov_dir
"""
}catch (Exception e) {
sh """
echo "${case_name} test faild print all log..."
for log in `ls /tmp/dm_test/*/*/log/*.log`; do
echo "____________________________________"
echo "\$log"
cat "\$log"
echo "____________________________________"
done
"""
throw e
}
}
stash includes: 'go/src/github.com/pingcap/dm/cov_dir/**', name: "integration-cov-${case_name}"
}
}
}
}


def run_single_it_test(String case_name) {
def label = 'dm-integration-test'
podTemplate(label: label,
Expand All @@ -212,7 +316,7 @@ def run_single_it_test(String case_name) {
envVar(key: 'MYSQL_ROOT_PASSWORD', value: "${MYSQL_PSWD}"),
],
args: "${MYSQL_ARGS}"),
// mysql 8.0
// mysql 8.0.21
containerTemplate(
name: 'mysql2', alwaysPullImage: false,
image: 'hub.pingcap.net/zhangxuecheng/mysql:8.0.21',ttyEnabled: true,
Expand Down Expand Up @@ -319,6 +423,7 @@ def run_make_coverage() {
unstash 'integration-cov-sharding2'
unstash 'integration-cov-ha'
unstash 'integration-cov-others'
unstash 'integration-cov-others_2'
} catch (Exception e) {
println e
}
Expand Down Expand Up @@ -657,7 +762,7 @@ pipeline {
stage('IT-tls') {
steps {
script {
run_single_it_test('tls')
run_tls_source_it_test('tls')
}
}
}
Expand Down Expand Up @@ -686,7 +791,15 @@ pipeline {
}
}

// END Integration Test

stage('IT-others-2') {
steps {
script {
run_single_it_test('others_2')
}
}
}
// END Integration Test
}
}

Expand All @@ -705,4 +818,4 @@ pipeline {
}
}
}
}
}