forked from vertica/vertica-testenv
-
Notifications
You must be signed in to change notification settings - Fork 0
/
vertica_testenv
executable file
·215 lines (183 loc) · 7.88 KB
/
vertica_testenv
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
#!/bin/bash
create_kdc(){
echo "Setting up KDC"
docker network create $NETWORK_NAME
docker build -t vertica/kdc $SRC/docker/kdc
docker run -d --privileged --name=$KDC_NAME --network=$NETWORK_NAME vertica/kdc
docker exec $KDC_NAME /kdc/install_kdc.sh
}
create_db(){
echo "Creating database"
if [ ! -f "$SRC/docker-vertica/packages/vertica-ce.latest.rpm" ]; then
export VERTICA_CE_URL="https://s3.amazonaws.com/vertica-community-edition-for-testing/XCz9cp7m/vertica-9.1.1-0.x86_64.RHEL6.rpm"
git clone https://github.com/jbfavre/docker-vertica.git
curl $VERTICA_CE_URL --create-dirs -o $SRC/docker-vertica/packages/vertica-ce.latest.rpm
fi
docker build -f $SRC/docker-vertica/Dockerfile.centos.7_9.x --build-arg VERTICA_PACKAGE=vertica-ce.latest.rpm -t jbfavre/vertica $SRC/docker-vertica
docker run -d -p 5433:5433 --name=$DB_NAME --mount type=bind,source="$(pwd)"/hdfs-config,target=/etc/hadoop/conf --network=$NETWORK_NAME jbfavre/vertica
echo "Making service keytab"
export V_PRINC=vertica/[email protected]
docker exec $KDC_NAME kadmin.local -q "addprinc -randkey $V_PRINC"
docker exec $KDC_NAME kadmin.local -q "ktadd -norandkey -k vertica.keytab $V_PRINC"
docker cp $KDC_NAME:vertica.keytab $SRC
docker cp $SRC/vertica.keytab $DB_NAME:/
rm $SRC/vertica.keytab
echo "Waiting for db to start"
sleep 60
echo "Kerberize db"
docker cp $SRC/docker/kerberize_db.sh $DB_NAME:/
docker exec $DB_NAME yum install -y krb5-workstation
docker exec $DB_NAME apt update
docker exec $DB_NAME apt install -y curl
docker exec $DB_NAME /bin/sh -c "echo $(docker inspect -f "{{.NetworkSettings.Networks.$NETWORK_NAME.IPAddress }}" $KDC_NAME) kerberos.example.com >> /etc/hosts"
docker exec $DB_NAME /bin/sh -c "echo $(docker inspect -f "{{.NetworkSettings.Networks.$NETWORK_NAME.IPAddress }}" $HDFS_NAME) hdfs.example.com hdfs >> /etc/hosts"
docker exec $DB_NAME /kerberize_db.sh
docker exec $DB_NAME mkdir -p /etc/pki/tls/certs/
docker exec $DB_NAME cp /etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt
docker exec $DB_NAME bash -c "cat /etc/hadoop/conf/hdfs.cert >> /etc/pki/tls/certs/ca-bundle.crt"
}
create_hdfs(){
echo "Creating HDFS"
docker run -d -i -p 22022:22 -p 8020:8020 -p 50010:50010 -p 50020:50020 -p 50070:50070 -p 50075:50075 -p 8088:8088 --name=$HDFS_NAME --mount type=bind,source="$(pwd)"/hdfs-config,target=/etc/hadoop/conf --network=$NETWORK_NAME mdouchement/hdfs
echo "Making service keytab"
export V_PRINC=root/[email protected]
export V_PRINC_H=HTTP/[email protected]
docker exec $KDC_NAME kadmin.local -q "addprinc -randkey $V_PRINC"
docker exec $KDC_NAME kadmin.local -q "addprinc -randkey $V_PRINC_H"
docker exec $KDC_NAME kadmin.local -q "ktadd -norandkey -k hdfs.keytab $V_PRINC $V_PRINC_H"
docker cp $KDC_NAME:hdfs.keytab $SRC
docker cp $SRC/hdfs.keytab $HDFS_NAME:/root/.keytab
rm $SRC/hdfs.keytab
echo "Waiting for HDFS to start"
sleep 60
docker exec $HDFS_NAME cp /etc/hadoop/conf/core-site.xml /opt/hadoop/etc/hadoop/core-site.xml
docker exec $HDFS_NAME cp /etc/hadoop/conf/hdfs-site.xml /opt/hadoop/etc/hadoop/hdfs-site.xml
docker exec $HDFS_NAME cp /etc/hadoop/conf/ssl-server.xml /opt/hadoop/etc/hadoop/ssl-server.xml
docker exec $HDFS_NAME cp /etc/hadoop/conf/keystore /root/.keystore
echo "Kerberize HDFS"
docker cp $SRC/docker/kerberize_hdfs.sh $HDFS_NAME:/
# docker exec $HDFS_NAME export PATH=$PATH:/usr/bin
docker exec $HDFS_NAME yum install -y krb5-workstation
docker exec $HDFS_NAME /bin/sh -c "echo /$(docker inspect -f "{{.NetworkSettings.Networks.$NETWORK_NAME.IPAddress }}" $HDFS_NAME)/d | sed /etc/hostsi > /etc/hosts2"
docker exec $HDFS_NAME /bin/sh -c "cat /etc/hosts2 > /etc/hosts"
docker exec $HDFS_NAME /bin/sh -c "echo $(docker inspect -f "{{.NetworkSettings.Networks.$NETWORK_NAME.IPAddress }}" $KDC_NAME) kerberos.example.com >> /etc/hosts"
docker exec $HDFS_NAME /bin/sh -c "echo $(docker inspect -f "{{.NetworkSettings.Networks.$NETWORK_NAME.IPAddress }}" $HDFS_NAME) hdfs.example.com hdfs >> /etc/hosts"
docker exec $HDFS_NAME /bin/sh /kerberize_hdfs.sh
docker exec $KDC_NAME /bin/sh -c "echo $(docker inspect -f "{{.NetworkSettings.Networks.$NETWORK_NAME.IPAddress }}" $HDFS_NAME) hdfs.example.com hdfs >> /etc/hosts"
echo "Restarting HDFS with proper config"
docker exec $HDFS_NAME stop-dfs.sh
docker exec $HDFS_NAME start-dfs.sh
}
create_sandbox(){
docker build -t vertica/sandbox $SRC/docker/sandbox
docker run --network=$NETWORK_NAME --mount type=bind,source="$(pwd)"/spark-connector,target=/connector --mount type=bind,source="$(pwd)"/hdfs-config,target=/etc/hadoop/conf -t -d --name=$SANDBOX_NAME vertica/sandbox
docker exec $SANDBOX_NAME /bin/sh -c "echo $(docker inspect -f "{{.NetworkSettings.Networks.$NETWORK_NAME.IPAddress }}" $KDC_NAME) kerberos.example.com >> /etc/hosts"
docker exec $SANDBOX_NAME /bin/sh -c "echo $(docker inspect -f "{{.NetworkSettings.Networks.$NETWORK_NAME.IPAddress }}" $DB_NAME) vertica.example.com vertica >> /etc/hosts"
docker exec $SANDBOX_NAME /bin/sh -c "echo $(docker inspect -f "{{.NetworkSettings.Networks.$NETWORK_NAME.IPAddress }}" $HDFS_NAME) hdfs.example.com hdfs >> /etc/hosts"
docker exec -it $SANDBOX_NAME /bin/bash
}
create_test(){
# Process python version
if [[ $PYENV =~ ^py[0-9][0-9]$ ]]; then
VER=${PYENV:2:1}.${PYENV:3}
cp $SRC/docker/test/Dockerfile $SRC/docker/test/Dockerfile.backup
sed -i "s/<ver>/$VER/g" $SRC/docker/test/Dockerfile
KDC_ADDR=$(docker inspect -f "{{.NetworkSettings.Networks.$NETWORK_NAME.IPAddress }}" $KDC_NAME)
DB_ADDR=$(docker inspect -f "{{.NetworkSettings.Networks.$NETWORK_NAME.IPAddress }}" $DB_NAME)
# Set up testing
docker build -f $SRC/docker/test/Dockerfile -t vertica/test --build-arg KDC_ADDR=$KDC_ADDR --build-arg DB_ADDR=$DB_ADDR $SRC/..
docker run --network=$NETWORK_NAME --rm --name=$TESTER_NAME vertica/test "-e $PYENV"
RESULT=$?
# cleanup
mv docker/test/Dockerfile.backup docker/test/Dockerfile
docker image rm vertica/test
exit $RESULT
else echo "Invalid python environment."
fi
}
stop_containers(){
echo "Stopping containers"
docker container stop $KDC_NAME
docker container stop $DB_NAME
docker container stop $HDFS_NAME
docker container stop $SANDBOX_NAME > /dev/null 2>&1
}
clean_system(){
echo "Cleaning up"
docker container rm $KDC_NAME
docker container rm $DB_NAME
docker container rm $HDFS_NAME
docker container rm $SANDBOX_NAME > /dev/null 2>&1
docker image rm jbfavre/vertica
docker image rm mdouchement/hdfs
docker image rm vertica/kdc
docker image rm vertica/sandbox > /dev/null 2>&1
docker network remove $NETWORK_NAME
}
echo_use(){
echo "Usage: $0 command [args]"
}
echo_help(){
echo -e '''
vertica_testenv sets up a kerberos-enabled vertica database to facilitate testing.
Usage: vertica_testenv command [args]
Commands: [start|test|stop|clean]
Start \t Build vertica and kerberos.
Test \t Build and run the vertica-python test suite.
Stop \t Stop the containers.
Clean \t Remove the containers and the images.
Options: --kdc, --db, --py
--kdc \t Name of the kdc container (default: vp.kdc)
--db \t Name of the database container (default: vp.db)
--py \t Python version. Options: py37 (default),py27,py34,py35,py36.
'''
}
if [ $# -eq 0 ]; then
echo_use
exit 0
fi
KDC_NAME=$USER.kdc
DB_NAME=$USER.db
HDFS_NAME=$USER.hdfs
TESTER_NAME=$USER.test
NETWORK_NAME=${USER}test
SANDBOX_NAME=${USER}.sandbox
PYENV=py37
SRC=$(dirname "$BASH_SOURCE")
op=$1
shift
while [ -n "$1" ]; do
case "$1" in
--help) echo_help
exit 0
;;
--kdc) KDC_NAME=$2
shift
;;
--db) DB_NAME=$2
shift
;;
--py) PYENV=$2
shift
;;
--) shift
break ;;
*) break ;;
esac
shift
done
if [ $op = 'start' ]; then
create_kdc
create_hdfs
create_db
elif [ $op = 'sandbox' ]; then
create_sandbox
elif [ $op = 'test' ]; then
create_test
elif [ $op = 'stop' ]; then
stop_containers
elif [ $op = 'clean' ]; then
clean_system
else
echo_use
fi