Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

scripts to run e2e against OpenShift 4 #2141

Merged
merged 2 commits into from
Apr 10, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions charts/tidb-cluster/templates/monitor-rbac.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,16 @@ rules:
resources:
- pods
verbs: ["get", "list", "watch"]
{{- if .Capabilities.APIVersions.Has "security.openshift.io/v1" }}
- apiGroups:
- security.openshift.io
resourceNames:
- anyuid
resources:
- securitycontextconstraints
verbs:
- use
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this is required because monitor-initializer must run as root (0)

{{- end }}
{{- if .Values.rbac.crossNamespace }}
- nonResourceURLs: ["/metrics"]
verbs: ["get"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ spec:
imagePullPolicy: {{ .Values.imagePullPolicy | default "IfNotPresent" }}
command:
- /usr/local/bin/tidb-admission-webhook
# use > 1024 port, then we can run it as non-root user
- --secure-port=6443
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
- --tls-cert-file=/var/serving-cert/tls.crt
- --tls-private-key-file=/var/serving-cert/tls.key
Expand All @@ -41,15 +43,15 @@ spec:
failureThreshold: 5
httpGet:
path: /healthz
port: 443
port: 6443
scheme: HTTPS
initialDelaySeconds: 5
timeoutSeconds: 5
readinessProbe:
failureThreshold: 5
httpGet:
path: /healthz
port: 443
port: 6443
scheme: HTTPS
initialDelaySeconds: 5
timeoutSeconds: 5
Expand All @@ -58,16 +60,23 @@ spec:
valueFrom:
fieldRef:
fieldPath: metadata.namespace
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
volumeMounts:
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
- mountPath: /var/serving-cert
name: serving-cert
{{- else }}
- mountPath: /apiserver.local.config
name: apiserver-local-config
{{- end }}
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
volumes:
{{- if eq .Values.admissionWebhook.apiservice.insecureSkipTLSVerify false }}
- name: serving-cert
secret:
defaultMode: 420
secretName: {{ .Values.admissionWebhook.apiservice.tlsSecret }}
{{- else }}
# rootfs maybe read-only, we need to an empty dir volume to store self-signed certifiates, etc.
- name: apiserver-local-config
emptyDir: {}
{{- end }}
{{- end }}
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ spec:
ports:
- name: https-webhook # optional
port: 443
targetPort: 443
targetPort: 6443
selector:
app.kubernetes.io/name: {{ template "chart.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
Expand Down
23 changes: 13 additions & 10 deletions charts/tidb-operator/templates/controller-manager-rbac.yaml
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
{{/*
Delete permission is required in OpenShift because we can't own resources we created if we can't delete them.
*/}}
{{- if .Values.rbac.create }}
kind: ServiceAccount
apiVersion: v1
Expand Down Expand Up @@ -29,16 +32,16 @@ rules:
verbs: ["*"]
- apiGroups: [""]
resources: ["endpoints","configmaps"]
verbs: ["create", "get", "list", "watch", "update"]
verbs: ["create", "get", "list", "watch", "update","delete"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["create","get","update"]
verbs: ["create","get","update","delete"]
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create", "update", "get", "list", "watch"]
verbs: ["create", "update", "get", "list", "watch","delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
Expand Down Expand Up @@ -83,10 +86,10 @@ Ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#privilege-esc
*/}}
- apiGroups: ["rbac.authorization.k8s.io"]
resources: [clusterroles,roles]
verbs: ["escalate","create","get","update"]
verbs: ["escalate","create","get","update", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["rolebindings","clusterrolebindings"]
verbs: ["create","get","update"]
verbs: ["create","get","update", "delete"]
---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1beta1
Expand Down Expand Up @@ -126,16 +129,16 @@ rules:
verbs: ["*"]
- apiGroups: [""]
resources: ["endpoints","configmaps"]
verbs: ["create", "get", "list", "watch", "update"]
verbs: ["create", "get", "list", "watch", "update", "delete"]
- apiGroups: [""]
resources: ["serviceaccounts"]
verbs: ["create","get","update"]
verbs: ["create","get","update","delete"]
- apiGroups: ["batch"]
resources: ["jobs"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
- apiGroups: [""]
resources: ["secrets"]
verbs: ["create", "update", "get", "list", "watch"]
verbs: ["create", "update", "get", "list", "watch", "delete"]
- apiGroups: [""]
resources: ["persistentvolumeclaims"]
verbs: ["get", "list", "watch", "create", "update", "delete"]
Expand All @@ -153,10 +156,10 @@ rules:
verbs: ["*"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["roles"]
verbs: ["escalate","create","get","update"]
verbs: ["escalate","create","get","update", "delete"]
- apiGroups: ["rbac.authorization.k8s.io"]
resources: ["rolebindings"]
verbs: ["create","get","update"]
verbs: ["create","get","update", "delete"]
{{- if .Values.features | has "AdvancedStatefulSet=true" }}
- apiGroups:
- apps.pingcap.com
Expand Down
159 changes: 159 additions & 0 deletions ci/run-in-vm.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,159 @@
#!/bin/bash

# Copyright 2020 PingCAP, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# See the License for the specific language governing permissions and
# limitations under the License.

#
# This is a helper script to start a VM and run command in it.
#
# TODO create an isolated network

set -o errexit
set -o nounset
set -o pipefail

ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
cd $ROOT

source "${ROOT}/hack/lib.sh"

GCP_CREDENTIALS=${GCP_CREDENTIALS:-}
GCP_PROJECT=${GCP_PROJECT:-}
GCP_ZONE=${GCP_ZONE:-}
GCP_SSH_PRIVATE_KEY=${GCP_SSH_PRIVATE_KEY:-}
GCP_SSH_PUBLIC_KEY=${GCP_SSH_PUBLIC_KEY:-}
NAME=${NAME:-tidb-operator-e2e}
GIT_URL=${GIT_URL:-https://github.com/pingcap/tidb-operator}
GIT_REF=${GIT_REF:-origin/master}
SYNC_FILES=${SYNC_FILES:-}

echo "GCP_CREDENTIALS: $GCP_CREDENTIALS"
echo "GCP_PROJECT: $GCP_PROJECT"
echo "GCP_ZONE: $GCP_ZONE"
echo "GCP_SSH_PRIVATE_KEY: $GCP_SSH_PRIVATE_KEY"
echo "GCP_SSH_PUBLIC_KEY: $GCP_SSH_PUBLIC_KEY"
echo "NAME: $NAME"
echo "GIT_URL: $GIT_URL"
echo "GIT_REF: $GIT_REF"
echo "SYNC_FILES: $SYNC_FILES"

# Pre-created nested virtualization enabled image with following commands:
#
# gcloud compute disks create disk1 --image-project centos-cloud --image-family centos-8 --zone us-central1-b
# gcloud compute images create centos-8-nested-vm \
# --source-disk disk1 --source-disk-zone us-central1-b \
# --licenses "https://compute.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx"
# gcloud compute disks delete disk1
#
# Refer to
# https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances
# for more details.
IMAGE=centos-8-nested-vm

echo "info: configure gcloud"
if [ -z "$GCP_PROJECT" ]; then
echo "error: GCP_PROJECT is required"
exit 1
fi
if [ -z "$GCP_CREDENTIALS" ]; then
echo "error: GCP_CREDENTIALS is required"
exit 1
fi
if [ -z "$GCP_ZONE" ]; then
echo "error: GCP_ZONE is required"
exit 1
fi
gcloud auth activate-service-account --key-file "$GCP_CREDENTIALS"
gcloud config set core/project $GCP_PROJECT
gcloud config set compute/zone $GCP_ZONE

echo "info: preparing ssh keypairs for GCP"
if [ ! -d ~/.ssh ]; then
mkdir ~/.ssh
fi
if [ ! -e ~/.ssh/google_compute_engine -a -n "$GCP_SSH_PRIVATE_KEY" ]; then
echo "Copying $GCP_SSH_PRIVATE_KEY to ~/.ssh/google_compute_engine" >&2
cp $GCP_SSH_PRIVATE_KEY ~/.ssh/google_compute_engine
chmod 0600 ~/.ssh/google_compute_engine
fi
if [ ! -e ~/.ssh/google_compute_engine.pub -a -n "$GCP_SSH_PUBLIC_KEY" ]; then
echo "Copying $GCP_SSH_PUBLIC_KEY to ~/.ssh/google_compute_engine.pub" >&2
cp $GCP_SSH_PUBLIC_KEY ~/.ssh/google_compute_engine.pub
chmod 0600 ~/.ssh/google_compute_engine.pub
fi

function gcloud_resource_exists() {
local args=($(tr -s '_' ' ' <<<"$1"))
unset args[$[${#args[@]}-1]]
local name="$2"
x=$(${args[@]} list --filter="name='$name'" --format='table[no-heading](name)' | wc -l)
[ "$x" -ge 1 ]
}

function gcloud_compute_instances_exists() {
gcloud_resource_exists ${FUNCNAME[0]} $@
}

function e2e::down() {
echo "info: tearing down"
if ! gcloud_compute_instances_exists $NAME; then
echo "info: instance '$NAME' does not exist, skipped"
return 0
fi
echo "info: deleting instance '$NAME'"
gcloud compute instances delete $NAME -q
}

function e2e::up() {
echo "info: setting up"
echo "info: creating instance '$NAME'"
gcloud compute instances create $NAME \
--machine-type n1-standard-8 \
--min-cpu-platform "Intel Haswell" \
--image $IMAGE \
--boot-disk-size 30GB \
--local-ssd interface=scsi
}

function e2e::test() {
echo "info: testing"
echo "info: syncing files $SYNC_FILES"
while IFS=$',' read -r line; do
IFS=':' read -r src dst <<< "$line"
if [ -z "$dst" ]; then
dst="$src"
fi
gcloud compute scp $src vagrant@$NAME:$dst
done <<< "$SYNC_FILES"
local tmpfile=$(mktemp)
trap "rm -f $tmpfile" RETURN
cat <<EOF > $tmpfile
sudo yum install -y git
cd \$HOME
sudo rm -rf tidb-operator
git init tidb-operator
cd tidb-operator
git fetch --tags --progress ${GIT_URL} +refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*
GIT_COMMIT=\$(git rev-parse ${GIT_REF}^{commit})
git checkout -f \${GIT_COMMIT}
$@
EOF
cat $tmpfile
gcloud compute scp $tmpfile vagrant@$NAME:/tmp/e2e.sh
gcloud compute ssh vagrant@$NAME --command "bash /tmp/e2e.sh"
}

e2e::down
trap 'e2e::down' EXIT
e2e::up
e2e::test "$@"
Loading