Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Commit

Permalink
Workflow Consistency / Simplification Updates in All Workflows - Part3 (
Browse files Browse the repository at this point in the history 
#336)

* Simplification workflow for docker and integTests

* tweak tar parameters
  • Loading branch information
@peterzhuamazon
peterzhuamazon committed Aug 11, 2020
1 parent 4a634d4 commit a5e49d5
Show file tree
Hide file tree
Showing 2 changed files with 273 additions and 237 deletions.
80 changes: 72 additions & 8 deletions .github/scripts/setup_runners_service.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,21 @@
# About: Setup ES/KIBANA for integTests on *NIX based ODFE distros w/wo Security
#
# Usage: ./setup_runners_service.sh $SETUP_DISTRO $SETUP_ACTION
# $SETUP_DISTRO: zip | deb | rpm (required)
# $SETUP_DISTRO: zip | docker | deb | rpm (required)
# $SETUP_ACTION: --es | --es-nosec | --kibana | --kibana-nosec (required)
#
# Requirements: This script assumes java 14 is already installed on the servers
#
# Starting Date: 2020-07-27
# Modified Date: 2020-08-02
# Modified Date: 2020-08-04
###############################################################################################

# This script allows users to manually assign parameters
if [ "$#" -ne 2 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ]
then
echo "Please assign 2 parameters when running this script"
echo "Example: $0 \$SETUP_DISTRO \$SETUP_ACTION"
echo "Example: $0 \"zip | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec\""
echo "Example: $0 \"zip | docker | deb | rpm\" \"--es | --es-nosec | --kibana | --kibana-nosec\""
exit 1
fi

Expand All @@ -43,13 +43,18 @@ ES_ROOT="${ROOT}/odfe-testing/${ES_PACKAGE_NAME}"
KIBANA_PACKAGE_NAME="opendistroforelasticsearch-kibana"
KIBANA_ROOT="${ROOT}/kibana-testing/${KIBANA_PACKAGE_NAME}"

DOCKER_NAME="Test-Docker-${OD_VERSION}"
DOCKER_NAME_KIBANA="Test-Docker-Kibana-${OD_VERSION}"
DOCKER_NAME_NoSec="Test-Docker-${OD_VERSION}-NoSec"
DOCKER_NAME_KIBANA_NoSec="Test-Docker-Kibana-${OD_VERSION}-NoSec"

S3_BUCKET="artifacts.opendistroforelasticsearch.amazon.com"

#####################################################################################################

echo "############################################################"
echo "Setup ES/KIBANA to start with YES/NO security configurations"
echo "User enters $SETUP_ACTION"
echo "User enters ${SETUP_DISTRO}:${SETUP_ACTION}"
echo "############################################################"

echo "setup parameters"
Expand All @@ -63,7 +68,14 @@ if [ "$SETUP_DISTRO" = "zip" ]
then
mkdir -p $ES_ROOT
aws s3 cp s3://$S3_BUCKET/downloads/tarball/opendistro-elasticsearch/$ES_PACKAGE_NAME.tar.gz . --quiet; echo $?
tar -zxf $ES_PACKAGE_NAME.tar.gz -C $ES_ROOT --strip 1
tar -zxf $ES_PACKAGE_NAME.tar.gz -C $ES_ROOT --strip-components 1
fi

if [ "$SETUP_DISTRO" = "docker" ]
then
echo "setup docker"
echo -n > Dockerfile
echo -n > Dockerfile.kibana
fi

if [ "$SETUP_DISTRO" = "deb" ]
Expand Down Expand Up @@ -93,10 +105,20 @@ echo "setup es"
# everything needs es
if [ "$SETUP_DISTRO" = "zip" ]
then
sed -i /install_demo_configuration/d $ES_ROOT/opendistro-tar-install.sh
sed -i /^node.max_local_storage_nodes/d $ES_ROOT/config/elasticsearch.yml
mkdir -p snapshots
echo "path.repo: [\"$PWD/snapshots\"]" >> $ES_ROOT/config/elasticsearch.yml
elif [ "$SETUP_DISTRO" = "docker" ]
then
echo "FROM opendistroforelasticsearch/opendistroforelasticsearch:$OD_VERSION" >> Dockerfile
echo "RUN echo '' >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile
echo "RUN echo \"path.repo: [\\\"/usr/share/elasticsearch\\\"]\" >> /usr/share/elasticsearch/config/elasticsearch.yml" >> Dockerfile
docker build -t odfe-http:security -f Dockerfile .
sleep 5
docker run -d -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME odfe-http:security
sleep 30
echo "Temp Solution to remove the wrong configuration. need be fixed in building stage"
docker exec -t $DOCKER_NAME /bin/bash -c "sed -i /^node.max_local_storage_nodes/d /usr/share/elasticsearch/config/elasticsearch.yml"
docker stop $DOCKER_NAME
else
sudo mkdir -p /home/repo
sudo chmod 777 /home/repo
Expand All @@ -111,6 +133,14 @@ then
then
cd $ES_ROOT
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
sleep 30
kill -9 `ps -ef | grep [e]lasticsearch | awk '{print $2}'`
sed -i /^node.max_local_storage_nodes/d ./config/elasticsearch.yml
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
elif [ "$SETUP_DISTRO" = "docker" ]
then
docker restart $DOCKER_NAME
docker ps
else
sudo systemctl restart elasticsearch.service
fi
Expand All @@ -132,8 +162,14 @@ then

if [ "$SETUP_DISTRO" = "zip" ]
then
sed -i /install_demo_configuration/d $ES_ROOT/opendistro-tar-install.sh
$ES_ROOT/bin/elasticsearch-plugin remove opendistro_security
sed -i '/http\.port/s/^# *//' $ES_ROOT/config/elasticsearch.yml
elif [ "$SETUP_DISTRO" = "docker" ]
then
echo "RUN /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security" >> Dockerfile
docker build -t odfe-http:no-security -f Dockerfile .
sleep 5
else
sudo /usr/share/elasticsearch/bin/elasticsearch-plugin remove opendistro_security
sudo sed -i '/http\.port/s/^# *//' /etc/elasticsearch/elasticsearch.yml
Expand All @@ -149,6 +185,10 @@ then
then
cd $ES_ROOT
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
elif [ "$SETUP_DISTRO" = "docker" ]
then
docker run -d -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME_NoSec odfe-http:no-security
docker ps
else
sudo systemctl restart elasticsearch.service
fi
Expand All @@ -172,7 +212,14 @@ then
then
mkdir -p $KIBANA_ROOT
aws s3 cp s3://$S3_BUCKET/downloads/tarball/$KIBANA_PACKAGE_NAME/$KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz . --quiet; echo $?
tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz -C $KIBANA_ROOT --strip 1
tar -zxf $KIBANA_PACKAGE_NAME-$OD_VERSION.tar.gz -C $KIBANA_ROOT --strip-components 1
elif [ "$SETUP_DISTRO" = "docker" ]
then
echo "FROM opendistroforelasticsearch/opendistroforelasticsearch-kibana:$OD_VERSION" >> Dockerfile.kibana
echo "RUN sed -i /^opendistro_security/d /usr/share/kibana/config/kibana.yml" >> Dockerfile.kibana
echo "RUN sed -i 's/https/http/' /usr/share/kibana/config/kibana.yml" >> Dockerfile.kibana
docker build -t odfe-kibana-http:security -f Dockerfile.kibana .
sleep 5
else
sudo apt install $KIBANA_PACKAGE_NAME=$OD_VERSION* -y || sudo yum install $KIBANA_PACKAGE_NAME-$OD_VERSION -y
fi
Expand All @@ -184,8 +231,17 @@ then
then
cd $ES_ROOT
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
sleep 30
kill -9 `ps -ef | grep [e]lasticsearch | awk '{print $2}'`
sed -i /^node.max_local_storage_nodes/d ./config/elasticsearch.yml
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
cd $KIBANA_ROOT
nohup ./bin/kibana > /dev/null 2>&1 &
elif [ "$SETUP_DISTRO" = "docker" ]
then
docker restart $DOCKER_NAME
docker run -d -p 5601:5601 --name $DOCKER_NAME_KIBANA --network="host" odfe-kibana-http:security
docker ps
else
sudo systemctl restart elasticsearch.service
sudo systemctl restart kibana.service
Expand Down Expand Up @@ -218,6 +274,14 @@ then
nohup ./opendistro-tar-install.sh > /dev/null 2>&1 &
cd $KIBANA_ROOT
nohup ./bin/kibana > /dev/null 2>&1 &
elif [ "$SETUP_DISTRO" = "docker" ]
then
echo "RUN /usr/share/kibana/bin/kibana-plugin remove opendistro_security" >> Dockerfile.kibana
docker build -t odfe-kibana-http:no-security -f Dockerfile.kibana .
sleep 5
docker run -d -p 9200:9200 -d -p 9600:9600 -e "discovery.type=single-node" --name $DOCKER_NAME_NoSec odfe-http:no-security
docker run -d -p 5601:5601 --name $DOCKER_NAME_KIBANA_NoSec --network="host" odfe-kibana-http:no-security
docker ps
else
sudo /usr/share/kibana/bin/kibana-plugin remove opendistro_security --allow-root
sudo sed -i /^opendistro_security/d /etc/kibana/kibana.yml
Expand Down
Loading

0 comments on commit a5e49d5

Please sign in to comment.