From f4d386ea45857914b61267b521989847ad96e05c Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Wed, 28 Apr 2021 17:42:31 -0400 Subject: [PATCH 1/5] New folder structure for opensearch-build repo Signed-off-by: Peter Zhu --- .gitignore | 12 + config/opensearch.yml | 7 + config/opensearch_dashboards.yml | 27 ++ docker/.DS_Store | Bin 0 -> 6148 bytes docker/build-image.sh | 103 +++++ .../opensearch-dashboards.al2.dockerfile | 92 +++++ docker/dockerfiles/opensearch.al2.dockerfile | 97 +++++ docker/opensearch-config/log4j2.properties | 9 + .../opensearch-docker-entrypoint.sh | 112 ++++++ .../opensearch-onetime-setup.sh | 50 +++ docker/opensearch-config/opensearch.yml | 7 + .../performance-analyzer.properties | 47 +++ ...opensearch-dashboards-docker-entrypoint.sh | 145 +++++++ .../opensearch.example.org.cert | 19 + .../opensearch.example.org.key | 27 ++ .../opensearch_dashboards.yml | 39 ++ opensearch/linux/opensearch-tar-install.sh | 73 ++++ scripts/setup_runners.sh | 264 +++++++++++++ tools/vulnerability-scan/wss-scan.config | 3 + tools/vulnerability-scan/wss-scan.sh | 105 +++++ .../wss-unified-agent.config | 363 ++++++++++++++++++ 21 files changed, 1601 insertions(+) create mode 100644 .gitignore create mode 100644 config/opensearch.yml create mode 100644 config/opensearch_dashboards.yml create mode 100644 docker/.DS_Store create mode 100755 docker/build-image.sh create mode 100644 docker/dockerfiles/opensearch-dashboards.al2.dockerfile create mode 100644 docker/dockerfiles/opensearch.al2.dockerfile create mode 100644 docker/opensearch-config/log4j2.properties create mode 100755 docker/opensearch-config/opensearch-docker-entrypoint.sh create mode 100755 docker/opensearch-config/opensearch-onetime-setup.sh create mode 100644 docker/opensearch-config/opensearch.yml create mode 100644 docker/opensearch-config/performance-analyzer.properties create mode 100755 docker/opensearch-dashboards-config/opensearch-dashboards-docker-entrypoint.sh create mode 100644 docker/opensearch-dashboards-config/opensearch.example.org.cert create mode 100644 docker/opensearch-dashboards-config/opensearch.example.org.key create mode 100644 docker/opensearch-dashboards-config/opensearch_dashboards.yml create mode 100755 opensearch/linux/opensearch-tar-install.sh create mode 100755 scripts/setup_runners.sh create mode 100644 tools/vulnerability-scan/wss-scan.config create mode 100755 tools/vulnerability-scan/wss-scan.sh create mode 100644 tools/vulnerability-scan/wss-unified-agent.config diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000000..a4ba72b808 --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +Thumbs.db +.DS_Store + +.idea/ +*.iml + +__pycache__ +.pytest_cache + +.gradle/ +.settings/ +out.txt diff --git a/config/opensearch.yml b/config/opensearch.yml new file mode 100644 index 0000000000..748b2235e5 --- /dev/null +++ b/config/opensearch.yml @@ -0,0 +1,7 @@ +cluster.name: docker-cluster + +# Bind to all interfaces because we don't know what IP address Docker will assign to us. +network.host: 0.0.0.0 + +# Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again. +discovery.type: single-node diff --git a/config/opensearch_dashboards.yml b/config/opensearch_dashboards.yml new file mode 100644 index 0000000000..c1b67e9490 --- /dev/null +++ b/config/opensearch_dashboards.yml @@ -0,0 +1,27 @@ +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +# Description: +# Default configuration for OpenSearch Dashboards + +opensearch.hosts: ["https://localhost:9200"] +opensearch.ssl.verificationMode: none +opensearch.username: "kibanaserver" +opensearch.password: "kibanaserver" +opensearch.requestHeadersWhitelist: [ authorization ] + +opensearch_security.multitenancy.enabled: true +opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"] +opensearch_security.readonly_mode.roles: ["kibana_read_only"] +# Use this setting if you are running kibana without https +opensearch_security.cookie.secure: false diff --git a/docker/.DS_Store b/docker/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..02ef0a302a9deb64100bdb27aac46745c8d3a172 GIT binary patch literal 6148 zcmeHK!Ab)$5S`SjTSV+Z&|@y%dT6P%Ct<02@FuM2L8b1xt1hk^>257rEBhPzM}C37 z<4lrPr1m6;l$kPllgUg%UP@*I0HV_m8vs=Ra8LNSG8Yi8SlJyV@Jx2sRxPllG zh^C_1@gEtWvs-}>6kNhBp4!*>Wx*i34-yq%#Cvd`#93aie~8jTd2wmkso-zbIrnbV zz{~x-pLP7+DfJHh{3fvWky6RT_JiX%8g#4cyDH87IE{KbISwNXIXj8dPz^e&pN5&9 z$1?-8PNiGj91dIUW=%Hhtx-)5+wFQyZZ}(_QN`KVYU~_b+&w-GpGPmRhGT($M=fg> z$MA;6jO{*ny);qjHAalH$61Wb05iZ0%ozjrnsZj>Y+t+`W`G&^Jp*( ztD|4)a1gFRZkYjQ;5!2grdy@@zd!r^|2>I&%m6bmR}6^qp10S*E!o;SwK=M_67>R= pgyIT=lN2;`DaKeT#ha*F&@ahAbS)MJ(SyPt0-6SHn1MfK;1iRKRLlSX literal 0 HcmV?d00001 diff --git a/docker/build-image.sh b/docker/build-image.sh new file mode 100755 index 0000000000..3195cc1a3d --- /dev/null +++ b/docker/build-image.sh @@ -0,0 +1,103 @@ +#!/bin/bash + +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + + +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + + +set -e + +function usage() { + echo "" + echo "This script is used to build the OpenSearch Docker image. It prepares the files required by the Dockerfile in a temporary directory, then builds and tags the Docker image." + echo "--------------------------------------------------------------------------" + echo "Usage: $0 [args]" + echo "" + echo "Required arguments:" + echo -e "-v VERSION\tSpecify the OpenSearch version number that you are building, e.g. '1.0.0' or '1.0.0-beta1'. This will be used to label the Docker image. If you do not use the '-o' option then this tool will download a public OPENSEARCH release matching this version." + echo -e "-f DOCKERFILE\tSpecify the dockerfile full path, e.g. dockerfile/opensearch.al2.dockerfile." + echo -e "-p PRODUCT\tSpecify the product, e.g. opensearch or opensearch-dashboards, make sure this is the name of your .tgz defined in dockerfile." + echo "" + echo "Optional arguments:" + echo -e "-o FILENAME\tSpecify a local OPENSEARCH tarball. You still need to specify the version - this tool does not attempt to parse the filename." + echo -e "-h\t\tPrint this message." + echo "--------------------------------------------------------------------------" +} + +while getopts ":ho:v:f:p:" arg; do + case $arg in + h) + usage + exit 1 + ;; + o) + TARBALL=`realpath $OPTARG` + ;; + v) + VERSION=$OPTARG + ;; + f) + DOCKERFILE=$OPTARG + ;; + p) + PRODUCT=$OPTARG + ;; + :) + echo "-${OPTARG} requires an argument" + usage + exit 1 + ;; + ?) + echo "Invalid option: -${arg}" + exit 1 + ;; + esac +done + +if [ -z "$VERSION" ] || [ -z "$DOCKERFILE" ] || [ -z "$PRODUCT" ]; then + echo "You must specify '-v VERSION', '-f DOCKERFILE', '-p PRODUCT'" + usage + exit 1 +fi + +echo $DOCKERFILE + +DIR=`mktemp -d` + +echo "Creating Docker workspace in $DIR" +trap '{ echo Removing Docker workspace in "$DIR"; rm -rf -- "$DIR"; }' TERM INT EXIT + +if [ -z "$TARBALL" ]; then + # No tarball file specified so download one + URL="https://artifacts.opensearch.org/releases/bundle/${PRODUCT}/${VERSION}/${PRODUCT}-${VERSION}-linux-x64.tar.gz" + echo "Downloading ${PRODUCT} version ${VERSION} from ${URL}" + curl -f $URL -o $DIR/$PRODUCT.tgz || exit 1 + ls -l $DIR +else + cp -v $TARBALL $DIR/$PRODUCT.tgz +fi + +cp -v ${PRODUCT}-config/* $DIR/ + +docker build --build-arg VERSION=$VERSION --build-arg BUILD_DATE=`date -u +%Y-%m-%dT%H:%M:%SZ` -f $DOCKERFILE $DIR -t opensearchproject/$PRODUCT:$VERSION + +rm -rf $DIR diff --git a/docker/dockerfiles/opensearch-dashboards.al2.dockerfile b/docker/dockerfiles/opensearch-dashboards.al2.dockerfile new file mode 100644 index 0000000000..b70b90669c --- /dev/null +++ b/docker/dockerfiles/opensearch-dashboards.al2.dockerfile @@ -0,0 +1,92 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + + +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + + +# This dockerfile generates an AmazonLinux-based image containing an OpenSearch-Dashboards installation. +# It assumes that the working directory contains four files: an OpenSearch-Dashboards tarball (opensearch-dashboards.tgz), opensearch_dashboards.yml, opensearch-dashboards-docker-entrypoint.sh, and example certs. +# Build arguments: +# VERSION: Required. Used to label the image. +# BUILD_DATE: Required. Used to label the image. Should be in the form 'yyyy-mm-ddThh:mm:ssZ', i.e. a date-time from https://tools.ietf.org/html/rfc3339. The timestamp must be in UTC. +# UID: Optional. Specify the opensearch-dashboards userid. Defaults to 1000. +# GID: Optional. Specify the opensearch-dashboards groupid. Defaults to 1000. +# OPENSEARCH_DASHBOARDS_HOME: Optional. Specify the opensearch-dashboards root directory. Defaults to /usr/share/opensearch-dashboards. + + +FROM amazonlinux:2 AS linux_x64_staging + +ARG UID=1000 +ARG GID=1000 +ARG OPENSEARCH_DASHBOARDS_HOME=/usr/share/opensearch-dashboards + +# Update packages +# Install the tools we need: tar and gzip to unpack the OpenSearch tarball, and shadow-utils to give us `groupadd` and `useradd`. +RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all + +# Create an opensearch-dashboards user, group, and directory +RUN groupadd -g $GID opensearch-dashboards && \ + adduser -u $UID -g $GID -d $OPENSEARCH_DASHBOARDS_HOME opensearch-dashboards && \ + mkdir /tmp/opensearch-dashboards + +# Prepare working directory +COPY opensearch-dashboards.tgz /tmp/opensearch-dashboards/opensearch-dashboards.tgz +RUN tar -xzf /tmp/opensearch-dashboards/opensearch-dashboards.tgz -C $OPENSEARCH_DASHBOARDS_HOME --strip-components=1 && rm -rf /temp/opensearch-dashboards +COPY opensearch-dashboards-docker-entrypoint.sh $OPENSEARCH_DASHBOARDS_HOME/ +COPY opensearch_dashboards.yml opensearch.example.org.* $OPENSEARCH_DASHBOARDS_HOME/config/ + +# Copy working directory to the actual release docker images +FROM amazonlinux:2 + +ARG UID=1000 +ARG GID=1000 +ARG OPENSEARCH_DASHBOARDS_HOME=/usr/share/opensearch-dashboards + +COPY --from=linux_x64_staging $OPENSEARCH_DASHBOARDS_HOME $OPENSEARCH_DASHBOARDS_HOME + +# Setup OpenSearch-dashboards +WORKDIR $OPENSEARCH_DASHBOARDS_HOME +RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all +RUN groupadd -g $GID opensearch-dashboards && \ + adduser -u $UID -g $GID -d $OPENSEARCH_DASHBOARDS_HOME opensearch-dashboards +RUN chown -R $UID:$GID $OPENSEARCH_DASHBOARDS_HOME + +# Change user +USER $UID + +# Expose port +EXPOSE 5601 + +ARG VERSION +ARG BUILD_DATE + +# Label +LABEL org.label-schema.schema-version="1.0" \ + org.label-schema.name="opensearch-dashboards" \ + org.label-schema.version="$VERSION" \ + org.label-schema.url="https://opensearch.org" \ + org.label-schema.vcs-url="https://github.com/opensearch-project/OpenSearch-Dashboards" \ + org.label-schema.license="Apache-2.0" \ + org.label-schema.vendor="Amazon" \ + org.label-schema.build-date="$BUILD_DATE" + +# CMD to run +CMD ["./opensearch-dashboards-docker-entrypoint.sh"] diff --git a/docker/dockerfiles/opensearch.al2.dockerfile b/docker/dockerfiles/opensearch.al2.dockerfile new file mode 100644 index 0000000000..747fc995d5 --- /dev/null +++ b/docker/dockerfiles/opensearch.al2.dockerfile @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + + +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + + +# This dockerfile generates an AmazonLinux-based image containing an OpenSearch installation. +# It assumes that the working directory contains four files: an OpenSearch tarball (opensearch.tgz), log4j2.properties, opensearch.yml, opensearch-docker-entrypoint.sh, opensearch-onetime-setup.sh. +# Build arguments: +# VERSION: Required. Used to label the image. +# BUILD_DATE: Required. Used to label the image. Should be in the form 'yyyy-mm-ddThh:mm:ssZ', i.e. a date-time from https://tools.ietf.org/html/rfc3339. The timestamp must be in UTC. +# UID: Optional. Specify the opensearch userid. Defaults to 1000. +# GID: Optional. Specify the opensearch groupid. Defaults to 1000. +# OPENSEARCH_HOME: Optional. Specify the opensearch root directory. Defaults to /usr/share/opensearch. +# PA_NAME: Optional. Specify the opensearch performance analyzer plugin folder name. Defaults to opensearch-performance-analyzer. + + +FROM amazonlinux:2 AS linux_x64_staging + +ARG UID=1000 +ARG GID=1000 +ARG OPENSEARCH_HOME=/usr/share/opensearch +ARG PA_NAME=opensearch-performance-analyzer + +# Update packages +# Install the tools we need: tar and gzip to unpack the OpenSearch tarball, and shadow-utils to give us `groupadd` and `useradd`. +RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all + +# Create an opensearch user, group, and directory +RUN groupadd -g $GID opensearch && \ + adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch && \ + mkdir /tmp/opensearch + +# Prepare working directory +COPY opensearch.tgz /tmp/opensearch/opensearch.tgz +RUN tar -xzf /tmp/opensearch/opensearch.tgz -C $OPENSEARCH_HOME --strip-components=1 && rm -rf /temp/opensearch +COPY opensearch-docker-entrypoint.sh opensearch-onetime-setup.sh $OPENSEARCH_HOME/ +COPY log4j2.properties opensearch.yml $OPENSEARCH_HOME/config/ +COPY performance-analyzer.properties $OPENSEARCH_HOME/plugins/$PA_NAME/pa_config/ + +# Copy working directory to the actual release docker images +FROM amazonlinux:2 + +ARG UID=1000 +ARG GID=1000 +ARG OPENSEARCH_HOME=/usr/share/opensearch +ARG PA_NAME=opensearch-performance-analyzer + +COPY --from=linux_x64_staging $OPENSEARCH_HOME $OPENSEARCH_HOME + +# Setup OpenSearch +WORKDIR $OPENSEARCH_HOME +RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all +RUN groupadd -g $GID opensearch && \ + adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch +RUN ./opensearch-onetime-setup.sh && \ + chown -R $UID:$GID $OPENSEARCH_HOME + +# Change user +USER $UID + +# Expose ports for the opensearch service (9200 for HTTP and 9300 for internal transport) and performance analyzer (9600 for the agent and 9650 for the root cause analysis component) +EXPOSE 9200 9300 9600 9650 + +ARG VERSION +ARG BUILD_DATE + +# Label +LABEL org.label-schema.schema-version="1.0" \ + org.label-schema.name="opensearch" \ + org.label-schema.version="$VERSION" \ + org.label-schema.url="https://opensearch.org" \ + org.label-schema.vcs-url="https://github.com/OpenSearch" \ + org.label-schema.license="Apache-2.0" \ + org.label-schema.vendor="Amazon" \ + org.label-schema.build-date="$BUILD_DATE" + +# CMD to run +CMD ["./opensearch-docker-entrypoint.sh"] diff --git a/docker/opensearch-config/log4j2.properties b/docker/opensearch-config/log4j2.properties new file mode 100644 index 0000000000..9ad290ad82 --- /dev/null +++ b/docker/opensearch-config/log4j2.properties @@ -0,0 +1,9 @@ +status = error + +appender.console.type = Console +appender.console.name = console +appender.console.layout.type = PatternLayout +appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n + +rootLogger.level = info +rootLogger.appenderRef.console.ref = console diff --git a/docker/opensearch-config/opensearch-docker-entrypoint.sh b/docker/opensearch-config/opensearch-docker-entrypoint.sh new file mode 100755 index 0000000000..305cf5c8da --- /dev/null +++ b/docker/opensearch-config/opensearch-docker-entrypoint.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + + +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +# This script specify the entrypoint startup actions for opensearch +# It will start both opensearch and performance analyzer plugin cli +# If either process failed, the entire docker container will be removed +# in favor of a newly started container + +# Files created by OpenSearch should always be group writable too +umask 0002 + +if [[ "$(id -u)" == "0" ]]; then + echo "OpenSearch cannot run as root. Please start your container as another user." + exit 1 +fi + +# Parse Docker env vars to customize OpenSearch +# +# e.g. Setting the env var cluster.name=testcluster +# +# will cause OpenSearch to be invoked with -Ecluster.name=testcluster + +declare -a opensearch_opts + +while IFS='=' read -r envvar_key envvar_value +do + # OpenSearch settings need to have at least two dot separated lowercase + # words, e.g. `cluster.name`, except for `processors` which we handle + # specially + if [[ "$envvar_key" =~ ^[a-z0-9_]+\.[a-z0-9_]+ || "$envvar_key" == "processors" ]]; then + if [[ ! -z $envvar_value ]]; then + opensearch_opt="-E${envvar_key}=${envvar_value}" + opensearch_opts+=("${opensearch_opt}") + fi + fi +done < <(env) + +# The virtual file /proc/self/cgroup should list the current cgroup +# membership. For each hierarchy, you can follow the cgroup path from +# this file to the cgroup filesystem (usually /sys/fs/cgroup/) and +# introspect the statistics for the cgroup for the given +# hierarchy. Alas, Docker breaks this by mounting the container +# statistics at the root while leaving the cgroup paths as the actual +# paths. Therefore, OpenSearch provides a mechanism to override +# reading the cgroup path from /proc/self/cgroup and instead uses the +# cgroup path defined the JVM system property +# es.cgroups.hierarchy.override. Therefore, we set this value here so +# that cgroup statistics are available for the container this process +# will run in. +export OPENSEARCH_JAVA_OPTS="-Dopensearch.cgroups.hierarchy.override=/ $OPENSEARCH_JAVA_OPTS" + + +# Start up the opensearch and performance analyzer agent processes. +# When either of them halts, this script exits, or we receive a SIGTERM or SIGINT signal then we want to kill both these processes. + +function terminateProcesses { + if kill -0 $OPENSEARCH_PID >& /dev/null; then + echo "Killing opensearch process $OPENSEARCH_PID" + kill -TERM $OPENSEARCH_PID + wait $OPENSEARCH_PID + fi + if kill -0 $PA_PID >& /dev/null; then + echo "Killing performance analyzer process $PA_PID" + kill -TERM $PA_PID + wait $PA_PID + fi +} + +# Enable job control so we receive SIGCHLD when a child process terminates +set -m + +# Make sure we terminate the child processes in the event of us received TERM (e.g. "docker container stop"), INT (e.g. ctrl-C), EXIT (this script terminates for an unexpected reason), or CHLD (one of the processes terminated unexpectedly) +trap terminateProcesses TERM INT EXIT CHLD + +# Export OpenSearch Home +export OPENSEARCH_HOME=/usr/share/opensearch + +# Start elasticsearch +$OPENSEARCH_HOME/bin/opensearch "${opensearch_opts[@]}" & +OPENSEARCH_PID=$! + +# Start performance analyzer agent +$OPENSEARCH_HOME/bin/performance-analyzer-agent-cli & +PA_PID=$! + +# Wait for the child processes to terminate +wait $OPENSEARCH_PID +echo "Elasticsearch exited with code $?" +wait $PA_PID +echo "Performance analyzer exited with code $?" diff --git a/docker/opensearch-config/opensearch-onetime-setup.sh b/docker/opensearch-config/opensearch-onetime-setup.sh new file mode 100755 index 0000000000..3f07ede5ea --- /dev/null +++ b/docker/opensearch-config/opensearch-onetime-setup.sh @@ -0,0 +1,50 @@ +#!/bin/bash + +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + + +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + + +# This script performs one-time setup for the OpenSearch tarball distribution. +# It installs a demo security config and sets up the performance analyzer + +OPENSEARCH_HOME=`dirname $(realpath $0)`; cd $OPENSEARCH_HOME + +##Security Plugin +SECURITY_PLUGIN="opensearch-security" +bash $OPENSEARCH_HOME/plugins/$SECURITY_PLUGIN/tools/install_demo_configuration.sh -y -i -s + +##Perf Plugin +PA_PLUGIN="opensearch-performance-analyzer" +chmod 755 $OPENSEARCH_HOME/plugins/$PA_PLUGIN/pa_bin/performance-analyzer-agent +chmod -R 755 /dev/shm +chmod 755 $OPENSEARCH_HOME/bin/performance-analyzer-agent-cli + +if ! grep -q '## OpenDistro Performance Analyzer' $OPENSEARCH_HOME/config/jvm.options; then + CLK_TCK=`/usr/bin/getconf CLK_TCK` + echo >> $OPENSEARCH_HOME/config/jvm.options + echo '## OpenDistro Performance Analyzer' >> $OPENSEARCH_HOME/config/jvm.options + echo "-Dclk.tck=$CLK_TCK" >> $OPENSEARCH_HOME/config/jvm.options + echo "-Djdk.attach.allowAttachSelf=true" >> $OPENSEARCH_HOME/config/jvm.options + echo "-Djava.security.policy=$OPENSEARCH_HOME/plugins/$PA_PLUGIN/pa_config/opensearch_security.policy" >> $OPENSEARCH_HOME/config/jvm.options +fi + diff --git a/docker/opensearch-config/opensearch.yml b/docker/opensearch-config/opensearch.yml new file mode 100644 index 0000000000..748b2235e5 --- /dev/null +++ b/docker/opensearch-config/opensearch.yml @@ -0,0 +1,7 @@ +cluster.name: docker-cluster + +# Bind to all interfaces because we don't know what IP address Docker will assign to us. +network.host: 0.0.0.0 + +# Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again. +discovery.type: single-node diff --git a/docker/opensearch-config/performance-analyzer.properties b/docker/opensearch-config/performance-analyzer.properties new file mode 100644 index 0000000000..72235cc9a8 --- /dev/null +++ b/docker/opensearch-config/performance-analyzer.properties @@ -0,0 +1,47 @@ +# ======================== OpenSearch performance analyzer plugin config ========================= + +# NOTE: this is an example for Linux. Please modify the config accordingly if you are using it under other OS. + +# Metrics data location +metrics-location = /dev/shm/performanceanalyzer/ + +# Metrics deletion interval (minutes) for metrics data. +# Interval should be between 1 to 60. +metrics-deletion-interval = 1 + +# If set to true, the system cleans up the files behind it. So at any point, we should expect only 2 +# metrics-db-file-prefix-path files. If set to false, no files are cleaned up. This can be useful, if you are archiving +# the files and wouldn't like for them to be cleaned up. +cleanup-metrics-db-files = true + +# WebService exposed by App's port +webservice-listener-port = 9600 + +# Port for RPC Communication +rpc-port = 9650 + +# Metric DB File Prefix Path location +metrics-db-file-prefix-path = /tmp/metricsdb_ + +https-enabled = false + +# Setup the correct path for server certificates +certificate-file-path = none +private-key-file-path = none +#trusted-cas-file-path = none + +# Setup the correct path for client certificates (by default, the client will just use the server certificates) +#client-certificate-file-path = specify_path +#client-private-key-file-path = specify_path +#client-trusted-cas-file-path = specify_path + +# WebService bind host; default only to local interface +webservice-bind-host = 0.0.0.0 + +# Plugin Stats Metadata file name, expected to be in the same location +plugin-stats-metadata = plugin-stats-metadata + +# Agent Stats Metadata file name, expected to be in the same location +agent-stats-metadata = agent-stats-metadata + + diff --git a/docker/opensearch-dashboards-config/opensearch-dashboards-docker-entrypoint.sh b/docker/opensearch-dashboards-config/opensearch-dashboards-docker-entrypoint.sh new file mode 100755 index 0000000000..ac5ad70b4c --- /dev/null +++ b/docker/opensearch-dashboards-config/opensearch-dashboards-docker-entrypoint.sh @@ -0,0 +1,145 @@ +#!/bin/bash + +# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +# +# Run Kibana, using environment variables to set longopts defining Kibana's +# configuration. +# +# eg. Setting the environment variable: +# +# ELASTICSEARCH_STARTUPTIMEOUT=60 +# +# will cause Kibana to be invoked with: +# +# --elasticsearch.startupTimeout=60 + +opensearch_dashboards_vars=( + console.enabled + console.proxyConfig + console.proxyFilter + elasticsearch.customHeaders + elasticsearch.logQueries + elasticsearch.password + elasticsearch.pingTimeout + elasticsearch.preserveHost + elasticsearch.requestHeadersWhitelist + elasticsearch.requestTimeout + elasticsearch.shardTimeout + elasticsearch.ssl.ca + elasticsearch.ssl.cert + elasticsearch.ssl.certificate + elasticsearch.ssl.certificateAuthorities + elasticsearch.ssl.key + elasticsearch.ssl.keyPassphrase + elasticsearch.ssl.verificationMode + elasticsearch.ssl.verify + elasticsearch.startupTimeout + elasticsearch.tribe.customHeaders + elasticsearch.tribe.password + elasticsearch.tribe.pingTimeout + elasticsearch.tribe.requestHeadersWhitelist + elasticsearch.tribe.requestTimeout + elasticsearch.tribe.ssl.ca + elasticsearch.tribe.ssl.cert + elasticsearch.tribe.ssl.certificate + elasticsearch.tribe.ssl.certificateAuthorities + elasticsearch.tribe.ssl.key + elasticsearch.tribe.ssl.keyPassphrase + elasticsearch.tribe.ssl.verificationMode + elasticsearch.tribe.ssl.verify + elasticsearch.tribe.url + elasticsearch.tribe.username + elasticsearch.hosts + kibana.defaultAppId + kibana.index + logging.dest + logging.quiet + logging.silent + logging.useUTC + logging.verbose + map.includeElasticMapsService + ops.interval + path.data + pid.file + regionmap + regionmap.includeElasticMapsService + server.basePath + server.customResponseHeaders + server.defaultRoute + server.host + server.maxPayloadBytes + server.name + server.port + server.rewriteBasePath + server.ssl.cert + server.ssl.certificate + server.ssl.certificateAuthorities + server.ssl.cipherSuites + server.ssl.clientAuthentication + server.customResponseHeaders + server.ssl.enabled + server.ssl.key + server.ssl.keyPassphrase + server.ssl.redirectHttpFromPort + server.ssl.supportedProtocols + server.xsrf.whitelist + status.allowAnonymous + status.v6ApiFormat + tilemap.options.attribution + tilemap.options.maxZoom + tilemap.options.minZoom + tilemap.options.subdomains + tilemap.url + timelion.enabled + vega.enableExternalUrls + opensearch_security.multitenancy.enabled + opensearch_security.multitenancy.tenants.preferred + opensearch_security.readonly_mode.roles +) + +longopts='' +for opensearch_dashboards_var in ${opensearch_dashboards_vars[*]}; do + # 'opensearch.hosts' -> 'OPENSEARCH_URL' + env_var=$(echo ${opensearch_dashboards_var^^} | tr . _) + + # Indirectly lookup env var values via the name of the var. + # REF: http://tldp.org/LDP/abs/html/bashver2.html#EX78 + value=${!env_var} + if [[ -n $value ]]; then + longopt="--${opensearch_dashboards_var}=${value}" + longopts+=" ${longopt}" + fi +done + +# Files created at run-time should be group-writable, for Openshift's sake. +umask 0002 + +# TO DO: +# Confirm with Mihir if this is necessary + +# The virtual file /proc/self/cgroup should list the current cgroup +# membership. For each hierarchy, you can follow the cgroup path from +# this file to the cgroup filesystem (usually /sys/fs/cgroup/) and +# introspect the statistics for the cgroup for the given +# hierarchy. Alas, Docker breaks this by mounting the container +# statistics at the root while leaving the cgroup paths as the actual +# paths. Therefore, OpenSearch-Dashboards provides a mechanism to override +# reading the cgroup path from /proc/self/cgroup and instead uses the +# cgroup path defined the configuration properties +# cpu.cgroup.path.override and cpuacct.cgroup.path.override. +# Therefore, we set this value here so that cgroup statistics are +# available for the container this process will run in. + +exec /usr/share/opensearch-dashboards/bin/opensearch-dashboards --cpu.cgroup.path.override=/ --cpuacct.cgroup.path.override=/ ${longopts} "$@" diff --git a/docker/opensearch-dashboards-config/opensearch.example.org.cert b/docker/opensearch-dashboards-config/opensearch.example.org.cert new file mode 100644 index 0000000000..9540047b98 --- /dev/null +++ b/docker/opensearch-dashboards-config/opensearch.example.org.cert @@ -0,0 +1,19 @@ +-----BEGIN CERTIFICATE----- +MIIDJTCCAg2gAwIBAgIJAKO3AgiyHbReMA0GCSqGSIb3DQEBBQUAMCkxJzAlBgNV +BAMMHm9wZW5kaXN0cm9mb3JlbGFzdGljc2VhcmNoLm9yZzAeFw0xOTAyMTUwNzI5 +MjNaFw0yOTAyMTIwNzI5MjNaMCkxJzAlBgNVBAMMHm9wZW5kaXN0cm9mb3JlbGFz +dGljc2VhcmNoLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALPO +w4iMMWj7wODNeAR+t9fFoFkhWmo7NiV9qteUtOTnOLBFG6VWVh2kQev1Ph2dF0uw +Dg632Pk3Tl1er+eKCoG+YzwkJqVZ0CJlxZdpN5jCpliCYxIaGUHO0I+Kdhzg1IUk +1Q2+8aeVN7ddiYZC13T/b5Kut8lr13O8OTqeBX2zRxuzX+jUNAZjAsCAdsv2jRrO +94883OyDzF6UXvETJcW5uF4j7NK9N2nfAplMy6Z7v/0kA89/ecuAb8wmf7MOul1h +sHyNd4aJwIGhKW/qBLJG+GHB5BiO6OaMJMzr5kzSV9we5r/plas5viAl/jWsYECS +rvGjUqBEOKOFy6RS2ZECAwEAAaNQME4wHQYDVR0OBBYEFI05BOGEHtUhrZmkIL3W +6KpJLE71MB8GA1UdIwQYMBaAFI05BOGEHtUhrZmkIL3W6KpJLE71MAwGA1UdEwQF +MAMBAf8wDQYJKoZIhvcNAQEFBQADggEBACvAn1fRvkJamVKdmsou+YTmxu5YhDva +MepBVT6yeqctcgQgUfaneuBrl/+9zmxkjykZ1uimSS2IM1FxIXuB8rX+iAcgFyIs +XXquOhJlaXs/kN0KBGUSRkTMQSGIaTh26U7GnCQrdaIZDtKp7CRWjNebeDucNOc+ +aXOqiNFbVePNe8+EZIDb0FmV4IiHuLcqCwzZvgesl3EUPuMsnrxeNKIw83ecpjqK +YRwWWhvY57asVbOZjn+M57bJSpzO9bO70uib0ItWs19aWqYP1vEOuNHEDbr2EVel +dejL2fYF06uiw179PSYnvDQohWdiqm8dM0Nx+QtJcbOR6BiBh8aQTjw= +-----END CERTIFICATE----- diff --git a/docker/opensearch-dashboards-config/opensearch.example.org.key b/docker/opensearch-dashboards-config/opensearch.example.org.key new file mode 100644 index 0000000000..e549011659 --- /dev/null +++ b/docker/opensearch-dashboards-config/opensearch.example.org.key @@ -0,0 +1,27 @@ +-----BEGIN RSA PRIVATE KEY----- +MIIEowIBAAKCAQEAs87DiIwxaPvA4M14BH6318WgWSFaajs2JX2q15S05Oc4sEUb +pVZWHaRB6/U+HZ0XS7AODrfY+TdOXV6v54oKgb5jPCQmpVnQImXFl2k3mMKmWIJj +EhoZQc7Qj4p2HODUhSTVDb7xp5U3t12JhkLXdP9vkq63yWvXc7w5Op4FfbNHG7Nf +6NQ0BmMCwIB2y/aNGs73jzzc7IPMXpRe8RMlxbm4XiPs0r03ad8CmUzLpnu//SQD +z395y4BvzCZ/sw66XWGwfI13honAgaEpb+oEskb4YcHkGI7o5owkzOvmTNJX3B7m +v+mVqzm+ICX+NaxgQJKu8aNSoEQ4o4XLpFLZkQIDAQABAoIBAEtPKXRyi2tWLlW5 +mTAAxw1Xl65dGtptjGMtfDWo8g8BjSu+9jJMs+rcVkP2xppupkgsJ+cgN39CG89d +jj1jAr2OuU+CMLGGqXcOuzCMgB26QSPHEKmPRs7aSV4ldwznWfueDqIgPGSdSeXC +Boy2SDmXluqPxShWw7mUtz+G2fPoQ+TMcH3puL3ooeV2iy25ZciLG3SFFQ7jTiJP +l6bNvnszVa7fRdapF25znC/XsfaW7XsXZwbV+NeWtO7CEIec/RExSy9N/abbmhWY +BPidBT9AQG4MoN10YunS7iwbfpXCNW0HKlFf77Mr5FrCZ0RE/Mg5JJrFF3Bt+sNT +v6/2+wUCgYEA3mEtewQCn40aIWYTTeXqtRCnpbt4y602B2PbY7OI/9N6xafATVnK ++bwcj9rKQRmUBbrvNGCY/GQJPqHRFjIRqK0y2QCavuR/980z96ojUXwsHG4eSmHY +wX0Rs+JtHVJDDH13hRWj/5Zxa+0ln2olBpHkp+o/VwiyYNXeqmw4Lq8CgYEAzv3p +kmOqNQpQ2y2jEdKYanIibsB27slb4UxBpCzEYSj3LF3CysaGF17GJRRCASH9Dm3Y +dZSgJzuBVZSgYl9HNSB1wY1gpHYeLm0F0yaE1D4tSoXGD+8pbLbl6LjwHOUtH4Yx +NCDwOv+pe6+tyO6v04xcAOSWWkcsHkesrcehi78CgYEAztNA8o3LRst8Rx3hzGyP +O7HuyoSvF3fUrVDhZjw/qfft0tFvWEGwN08OnXjW7W81rZKsEWHaR7teocLsbSFK +LUBQ72TRh8OLhVR0Bd9Wk93SROrh/zo719FT2qM36rJsq7wxojR7YCZy/PcGK9sT +7qVYLhf+qxrqlIe1XXnjUv8CgYBhglm63Xp8HAz1A22zYqrsUjIITGB6op7G5p5x +kak9QAgYfV7bGWRns/zR0NY7JGDKoFJ9KXO8p5qIz0gyr0GjNFRkxo4FS6NyKWe6 +DYY7WcMKPTSNoxO3o/1wgVaTth5ooFWjgfTJRlD39E2Av3hSxuc8gHKXurl81jTx +dH6UjwKBgEA5JLPaICkkY4ppcwdnf7zFFi22Rq5e2AJQSJcHOgE0K8/jJqbv+VOy +ugVjXRpq6VdNjGD3aLToe/SnRLUDmq+ZJqV+BoZSv3DJCsUkc4O9tS/E6Tn4A1di +loi+vT/GFEVVE3MaB42XGaaC82VEgwdqE3hDqNM/F2i3DfyoKD6V +-----END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/docker/opensearch-dashboards-config/opensearch_dashboards.yml b/docker/opensearch-dashboards-config/opensearch_dashboards.yml new file mode 100644 index 0000000000..f34b782048 --- /dev/null +++ b/docker/opensearch-dashboards-config/opensearch_dashboards.yml @@ -0,0 +1,39 @@ +--- +# SPDX-License-Identifier: Apache-2.0 +# +# The OpenSearch Contributors require contributions made to +# this file be licensed under the Apache-2.0 license or a +# compatible open source license. +# +# Modifications Copyright OpenSearch Contributors. See +# GitHub history for details. + + +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +# Description: +# Default configuration for OpenSearch Dashboards + +opensearch.hosts: ["http://localhost:9200"] +opensearch.ssl.verificationMode: full +opensearch.username: "kibanaserver" +opensearch.password: "kibanaserver" +opensearch.requestHeadersWhitelist: [ authorization ] + +opensearch_security.multitenancy.enabled: true +opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"] +opensearch_security.readonly_mode.roles: ["kibana_read_only"] +# Use this setting if you are running kibana without https +opensearch_security.cookie.secure: false + diff --git a/opensearch/linux/opensearch-tar-install.sh b/opensearch/linux/opensearch-tar-install.sh new file mode 100755 index 0000000000..d51781facf --- /dev/null +++ b/opensearch/linux/opensearch-tar-install.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +OPENSEARCH_HOME=`dirname $(realpath $0)`; cd $OPENSEARCH_HOME +KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/knnlib +##Security Plugin +bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s + +##Perf Plugin +chmod 755 $OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_bin/performance-analyzer-agent +chmod -R 755 /dev/shm +chmod 755 $OPENSEARCH_HOME/bin/performance-analyzer-agent-cli +echo "done security" +PA_AGENT_JAVA_OPTS="-Dlog4j.configurationFile=$OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_config/log4j2.xml \ + -Xms64M -Xmx64M -XX:+UseSerialGC -XX:CICompilerCount=1 -XX:-TieredCompilation -XX:InitialCodeCacheSize=4096 \ + -XX:InitialBootClassLoaderMetaspaceSize=30720 -XX:MaxRAM=400m" + +OPENSEARCH_MAIN_CLASS="com.amazon.opendistro.opensearch.performanceanalyzer.PerformanceAnalyzerApp" \ +OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=plugins/opensearch-performance-analyzer \ +OPENSEARCH_JAVA_OPTS=$PA_AGENT_JAVA_OPTS + +if ! grep -q '## OpenSearch Performance Analyzer' $OPENSEARCH_HOME/config/jvm.options; then + CLK_TCK=`/usr/bin/getconf CLK_TCK` + echo >> $OPENSEARCH_HOME/config/jvm.options + echo '## OpenSearch Performance Analyzer' >> $OPENSEARCH_HOME/config/jvm.options + echo "-Dclk.tck=$CLK_TCK" >> $OPENSEARCH_HOME/config/jvm.options + echo "-Djdk.attach.allowAttachSelf=true" >> $OPENSEARCH_HOME/config/jvm.options + echo "-Djava.security.policy=$OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_config/opensearch_security.policy" >> $OPENSEARCH_HOME/config/jvm.options +fi +echo "done plugins" + +##Check KNN lib existence in OpenSearch TAR distribution +echo "Checking kNN library" +FILE=`ls $KNN_LIB_DIR/libKNNIndex*.so` +if test -f "$FILE"; then + echo "FILE EXISTS $FILE" +else + echo "TEST FAILED OR FILE NOT EXIST $FILE" +fi + +##Set KNN Dylib Path for macOS and *nix systems +if echo "$OSTYPE" | grep -qi "darwin"; then + if echo "$JAVA_LIBRARY_PATH" | grep -q "$KNN_LIB_DIR"; then + echo "KNN lib path has been set" + else + export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:$KNN_LIB_DIR + echo "KNN lib path not found, set new path" + echo $JAVA_LIBRARY_PATH + fi +else + if echo "$LD_LIBRARY_PATH" | grep -q "$KNN_LIB_DIR"; then + echo "KNN lib path has been set" + else + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$KNN_LIB_DIR + echo "KNN lib path not found, set new path" + echo $LD_LIBRARY_PATH + fi +fi + +##Start OpenSearch +bash $OPENSEARCH_HOME/bin/opensearch "$@" diff --git a/scripts/setup_runners.sh b/scripts/setup_runners.sh new file mode 100755 index 0000000000..d042a326ea --- /dev/null +++ b/scripts/setup_runners.sh @@ -0,0 +1,264 @@ +#!/bin/bash + +###### Information ############################################################################ +# Name: setup_runners.sh +# +# About: 1. Run instances on EC2 based on parameters defined and wait for completion +# 2. SSH to these instances and configure / bootstrap on $GIT_URL_REPO as runners +# 3. Unbootstrap the runners and terminate the instances for cleanups +# +# Usage: ./setup_runners.sh $ACTION $EC2_INSTANCE_NAMES $GITHUB_TOKEN +# $ACTION: run | terminate (required) +# $EC2_INSTANCE_NAMES: (required, sep ",") +# $GITHUB_TOKEN: GitHub PAT with repo scope and Admin Access to $GIT_URL_REPO +# +# Requirements: The env that runs this script must have its AWS resources with these configurations +# +# 1. Have an AWS user account with access to EC2 resource, remember the User ID +# +# 2. Create EC2 keypairs with name "odfe-release-runner" +# +# 3. Create EC2 Security Group with name "odfe-release-runner" +# with inbound rules of 22/9200/9600/5601 from IP ranges that need access to the runner +# +# 4. Create IAM resources: +# +# * IAM role with name "odfe-release-runner", and these policies attached to it: +# i. AmazonEC2RoleforSSM +# ii. AmazonSSMManagedInstanceCore +# +# * IAM user "opendistro-ec2-user", generate a pair of security credentials, +# and these policies attached to it: +# i. AmazonEC2FullAccess +# ii. Custom policy using this json, I name it again to "odfe-release-runner" +# { +# "Version": "2012-10-17", +# "Statement": [ +# { +# "Sid": "VisualEditor0", +# "Effect": "Allow", +# "Action": [ +# "ssm:SendCommand", +# "iam:PassRole" +# ], +# "Resource": [ +# "arn:aws:ssm:*:*:document/*", +# "arn:aws:ec2:*:*:instance/*", +# "arn:aws:iam:::role/" +# ] +# }, +# { +# "Sid": "VisualEditor1", +# "Effect": "Allow", +# "Action": "ssm:DescribeInstanceInformation", +# "Resource": "*" +# } +# ] +# } +# +# 5. awscli must "aws login" with the security credencial created for IAM user +# in the step 4 above +# +# 6. If you change the above resources name from "odfe-release-runner" to "xyz", +# please update "Variables / Parameters / Settings" section of this script +# +# 7. Runner AMI requires installation of packages of these (java version can be different as gradle might request a higher version): +# Debian: +# sudo apt install -y curl wget unzip tar jq python python3 git awscli openjdk-14-jdk +# sudo apt install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb +# +# RedHat: +# sudo yum install -y curl wget unzip tar jq python python3 git awscli java-latest-openjdk +# sudo yum install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib +# +# Also you need to install java devel if you want to compile library (e.g. knnlib) +# +# 8. AMI must be at least 16GB during the creation. +# +# 9. You can use `export GIT_UTL_REPO="opendistro-for-elasticsearch/opendistro-build"` or similar to set the Git Repo of the runner +# +# 10. JDK & SSM Agent +# You should find a way to install JDK14 or later on the server +# Dibian with: sudo add-apt-repository ppa:openjdk-r/ppa +# RedHat with: https://fedoraproject.org/wiki/EPEL +# +# Also, you need to install ssm agent +# on non-al2 machine due to ssm RunCommand requires that +# https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-manual-agent-install.html +# +# us-west-2 +# RPM x64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/linux_amd64/amazon-ssm-agent.rpm +# RPM arm64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/linux_arm64/amazon-ssm-agent.rpm +# DEB x64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/debian_amd64/amazon-ssm-agent.deb +# DEB arm64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/debian_arm64/amazon-ssm-agent.deb +# yum or dpkg then systemctl enable/start amazon-ssm-agent +# +# 11. You also need to set the user of the GitHub Token to have ADMIN access of the GitHub Repo +# So that runner can be successfully bootstrapped to action tab in settings. +# +############################################################################################### + +set -e + +##################################### +# Variables / Parameters / Settings # +##################################### + +# This script allows users to manually assign parameters +if [ "$#" -lt 3 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ] +then + echo "Please assign at least 3 parameters when running this script" + echo "Example: $0 \$ACTION \$EC2_INSTANCE_NAMES(,) \$GITHUB_TOKEN, \$EC2_AMI_ID" + echo "Example (run must have 4 parameters): $0 \"run\" \"opensearch-rpm-im,opensearch-rpm-sql\" \"\" \"ami-*\"" + echo "Example (terminate must have 3 parameters): $0 \"terminate\" \"opensearch-rpm-im,opensearch-rpm-sql\" \"\"" + echo "You can use \`export GIT_UTL_REPO=\"opendistro-for-elasticsearch/opendistro-build\"\` or similar to set the Git Repo of the runner" + exit 1 +fi + +SETUP_ACTION=$1 +SETUP_RUNNER=`echo $2 | sed 's/,/ /g'` +SETUP_GIT_TOKEN=$3 + +# AMI on us-west-2 +# Distro Arch Recommand Username AMI-ID Java Comments +# RPM-al2 x64 YES ec2-user ami-0bd968fea932935f4 none no jdk + reports kibana dependencies +# RPM-al2 arm64 YES ec2-user ami-0ef0c96643bbd01f2 jdk14 preinstall with tar.gz + reports kibana dependencies +# DEB-ubu1804 arm64 YES ubuntu ami-03f8a33a16290a84c jdk14 preinstall + docker + docker compose + reports kibana dependencies +# RPM-centos8 x64 NO centos ami-011f59f50bac33376 jdk15 preinstall +# RPM-centos8 arm64 NO centos ami-0ed17173ab64255b1 jdk15 preinstall +EC2_AMI_ID=$4 + +if [ "$SETUP_ACTION" = "run" ] +then + if [ -z "$EC2_AMI_ID" ] + then + echo " \$EC2_AMI_ID is empty, please add a 4th parameter for the run " + exit 1 + else + # This does not support MacOS now due to cumbersome descriptions + # MacOS sample: ami-00b3e436dc75183e0 + # "PlatformDetails": "Linux/UNIX" + # "Architecture": "x86_64_mac" + EC2_AMI_PLATFORM=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].PlatformDetails' --output text | awk -F '/' '{print $1}' | tr '[:upper:]' '[:lower:]'` + EC2_AMI_ARCH=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].Architecture' --output text | sed 's/x86_64/x64/g'` + EC2_AMI_NAME=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].Name' --output text | tr '[:upper:]' '[:lower:]'` + EC2_AMI_USER="ec2-user"; if echo $EC2_AMI_NAME | grep "centos"; then EC2_AMI_USER="centos"; elif echo $EC2_AMI_NAME | grep "ubuntu"; then EC2_AMI_USER="ubuntu"; fi + EC2_INSTANCE_TYPE="m5.xlarge"; if [ "$EC2_AMI_ARCH" = "arm64" ]; then EC2_INSTANCE_TYPE="m6g.xlarge"; fi + RUNNER_URL=`curl -s https://api.github.com/repos/actions/runner/releases/latest -H "Authorization: token $SETUP_GIT_TOKEN" | jq -r '.assets[].browser_download_url' | grep "$EC2_AMI_PLATFORM" | grep "$EC2_AMI_ARCH" | tail -n 1` + echo Provision $EC2_AMI_PLATFORM $EC2_AMI_ARCH $EC2_AMI_NAME $EC2_AMI_USER $EC2_INSTANCE_TYPE $RUNNER_URL + fi +fi + + +EC2_INSTANCE_SIZE=20 #GiB +EC2_KEYPAIR="odfe-release-runner" +EC2_SECURITYGROUP="odfe-release-runner" +IAM_ROLE="odfe-release-runner" +GIT_URL_API="https://api.github.com/repos" +GIT_URL_BASE="https://github.com" +GIT_URL_REPO=${GIT_URL_REPO:-opensearch-project/opensearch-build} +RUNNER_DIR="actions-runner" + + +echo "###############################################" +echo "Start Running $0 $1 $2" +echo "###############################################" + +############################################### +# Run / Start instances and bootstrap runners # +############################################### +if [ "$SETUP_ACTION" = "run" ] +then + echo "GIT_URL_REPO $GIT_URL_REPO" + + echo "" + echo "Run / Start instances and bootstrap runners [${SETUP_RUNNER}]" + echo "" + + # Get information + instance_root_device=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].RootDeviceName' --output text` + + # Provision VMs + for instance_name1 in $SETUP_RUNNER + do + echo "[${instance_name1}]: Start provisioning vm" + aws ec2 run-instances --image-id $EC2_AMI_ID --count 1 --instance-type $EC2_INSTANCE_TYPE \ + --block-device-mapping DeviceName=$instance_root_device,Ebs={VolumeSize=$EC2_INSTANCE_SIZE} \ + --key-name $EC2_KEYPAIR --security-groups $EC2_SECURITYGROUP \ + --iam-instance-profile Name=$IAM_ROLE \ + --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$instance_name1}]" > /dev/null 2>&1; echo $? + sleep 1 + done + + echo "" + echo "Sleep for 120 seconds for EC2 instances to start running" + echo "" + + sleep 120 + + # Setup VMs to register as runners + for instance_name2 in $SETUP_RUNNER + do + echo "[${instance_name2}]: Make change of the runner hostname" + aws ssm send-command --targets Key=tag:Name,Values=$instance_name2 --document-name "AWS-RunShellScript" \ + --parameters '{"commands": ["#!/bin/bash", "sudo hostnamectl set-hostname '${instance_name2}'"]}' \ + --output text > /dev/null 2>&1; echo $? + + echo "[${instance_name2}]: Get latest runner binary to server ${RUNNER_URL}" + aws ssm send-command --targets Key=tag:Name,Values=$instance_name2 --document-name "AWS-RunShellScript" \ + --parameters '{"commands": ["#!/bin/bash", "sudo su - '${EC2_AMI_USER}' -c \"mkdir -p '${RUNNER_DIR}' && cd '${RUNNER_DIR}' && wget -q '${RUNNER_URL}' && tar -xzf *.tar.gz && rm *.tar.gz \""]}' \ + --output text > /dev/null 2>&1; echo $? + + echo "[${instance_name2}]: Get runner token and bootstrap on Git" + instance_runner_token=`curl --silent -H "Authorization: token ${SETUP_GIT_TOKEN}" --request POST "${GIT_URL_API}/${GIT_URL_REPO}/actions/runners/registration-token" | jq -r .token` + # Wait 10 seconds for untar of runner binary to complete + aws ssm send-command --targets Key=tag:Name,Values=$instance_name2 --document-name "AWS-RunShellScript" \ + --parameters '{"commands": ["#!/bin/bash", "sudo su - '${EC2_AMI_USER}' -c \"sleep 30 && cd '${RUNNER_DIR}' && ./config.sh --unattended --url '${GIT_URL_BASE}/${GIT_URL_REPO}' --labels '${instance_name2}' --token '${instance_runner_token}' && nohup ./run.sh &\""]}' \ + --output text > /dev/null 2>&1; echo $? + sleep 5 + done + + echo "" + echo "Wait for 90 seconds for runners to bootstrap on Git" + echo "" + + sleep 90 + + echo "" + echo "All runners are online on Git" + echo "" +fi + + +################################################### +# Terminate / Delete instances and remove runners # +################################################### +if [ "$SETUP_ACTION" = "terminate" ] +then + echo "GIT_URL_REPO $GIT_URL_REPO" + + echo "" + echo "Terminate / Delete instances and remove runners [${SETUP_RUNNER}]" + echo "" + + for instance_name3 in $SETUP_RUNNER + do + instance_runner_id_git=`curl --silent -H "Authorization: token ${SETUP_GIT_TOKEN}" --request GET "${GIT_URL_API}/${GIT_URL_REPO}/actions/runners" | jq ".runners[] | select(.name == \"${instance_name3}\") | .id"` + echo "[${instance_name3}]: Unbootstrap runner from Git" + curl --silent -H "Authorization: token ${SETUP_GIT_TOKEN}" --request DELETE "${GIT_URL_API}/${GIT_URL_REPO}/actions/runners/${instance_runner_id_git}"; echo $? + + instance_runner_id_ec2=`aws ec2 describe-instances --filters "Name=tag:Name,Values=$instance_name3" | jq -r '.Reservations[].Instances[] | select(.State.Code == 16) | .InstanceId'` # Only running instances + echo "[${instance_name3}]: Remove tags Name" + aws ec2 delete-tags --resources $instance_runner_id_ec2 --tags Key=Name > /dev/null 2>&1; echo $? + + echo "[${instance_name3}]: Terminate runner" + aws ec2 terminate-instances --instance-ids $instance_runner_id_ec2 > /dev/null 2>&1; echo $? + + sleep 1 + done + + echo "All runners are offline on Git" +fi + + + diff --git a/tools/vulnerability-scan/wss-scan.config b/tools/vulnerability-scan/wss-scan.config new file mode 100644 index 0000000000..30d7e64d2f --- /dev/null +++ b/tools/vulnerability-scan/wss-scan.config @@ -0,0 +1,3 @@ +baseDirPath=$(pwd) +gitBasePath=https://github.com/opensearch-project/ +gitRepos=alerting,alerting-dashboards-plugin,anomaly-detection,anomaly-detection-dashboards-plugin,asynchronous-search,common-utils,dashboards-notebooks,dashboards-reports,dashboards-visualizations,data-prepper,index-management,index-management-dashboards-plugin,job-scheduler,k-NN,opensearch-cli,performance-analyzer,performance-analyzer-rca,perftop,security,security-dashboards-plugin,sql,trace-analytics diff --git a/tools/vulnerability-scan/wss-scan.sh b/tools/vulnerability-scan/wss-scan.sh new file mode 100755 index 0000000000..97c9c318b6 --- /dev/null +++ b/tools/vulnerability-scan/wss-scan.sh @@ -0,0 +1,105 @@ +#!/bin/bash +###### Information ############################################################################ +# Name: wss-scan.sh +# Language: Shell +# +# About: This script is to scan the OpenSearch distros for vulnerabilities and licenses +# It will scan the repositories and send the WhiteSource link to the mail +# of the user. +# +# Prerequisites: Need to install Java 11 +# Export JAVA_HOME env variable to the JDK path +# Add JAVA_HOME to PATH variable +# Need to set the recepient mail in wss-scan.config for local run +# WhiteSource API key is needed for local run, The API Key can be retrieved from the +# WhiteSource Admin Console of your account.Use the below command to export the API key +# export wss_apikey=$(APIKEY) +# +# Usage: ./wss-scan.sh +# +############################################################################################### + +set -e + +java -version +if [ "$?" != 0 ] +then + echo "Java has not been setup" + exit 1 +fi + +if [ ! -f "wss-unified-agent.jar" ] +then + # Download the WhiteSource Agent + wget -q https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar + # The version 20.9.2.1 has been tested and can be used if a specific version is required + #wget -q https://github.com/whitesource/unified-agent-distribution/releases/download/v20.9.2.1/wss-unified-agent.jar +fi + +# scan the config file for the user configurations +# wss-scan.config has to be present in the same working directory as the script +source wss-scan.config + +# change comma to whitespace +gitRepos=${gitRepos//,/$'\n'} + +basepath=$baseDirPath"/repos" + +echo "Cleaning up scan directories if already present" +rm -rf $basepath + +mkdir -p $basepath + + +# clone the desired Repos for scanning +for repo in $gitRepos +do + echo "Cloning repo "$gitBasePath$repo + git clone "$gitBasePath$repo".git $basepath"/"$repo +done + +echo -n > info.txt + + +# scan the Repos using the WhiteSource Unified Agent +for repo in $gitRepos +do + repo_path=$basepath"/"$repo + if [ -d "$repo_path" ] + then + echo "Scanning repo: "$gitBasePath$repo " Project: " $repo + java -jar wss-unified-agent.jar -c wss-unified-agent.config -d $repo_path -apiKey $wss_apikey -product OpenSearch -project $repo | grep "Project name" | sed 's/^.\{,41\}//' >> info.txt 2>&1 + else + echo "Scanning failed for repo: "$gitBasePath$repo " Project: " $repo + fi +done + + + +# mail function to send the scan details to the desired recepient +mail_format_func() +{ + +echo "" > output.md +while IFS= read -r line +do +# setting comma as the delimiter + + IFS=',' + read -ra val <<< "$line" + echo "" >> output.md + for ln in "${val[@]}" + do + echo "${ln//[[:space:]]/}" + echo "" >> output.md + done + echo "" >> output.md +done < info.txt +echo "
"${ln//[[:space:]]/}"
" >> output.md + +} + +mail_format_func + +# remove the WhiteSource unified Jar +rm "wss-unified-agent.jar" diff --git a/tools/vulnerability-scan/wss-unified-agent.config b/tools/vulnerability-scan/wss-unified-agent.config new file mode 100644 index 0000000000..6d50d45e7e --- /dev/null +++ b/tools/vulnerability-scan/wss-unified-agent.config @@ -0,0 +1,363 @@ +############################################################### +# WhiteSource Unified-Agent configuration file +############################################################### +# GENERAL SCAN MODE: Files and Package Managers +############################################################### +# Organization vitals +###################### + +#apiKey='${wss_apikey}' +apiKey= +#userKey is required if WhiteSource administrator has enabled "Enforce user level access" option +#userKey= +#requesterEmail=user@provider.com + +projectName= +projectVersion= +projectToken= +#projectTag= key:value + +productName= +productVersion= +productToken= + +#projectPerFolder=true +#projectPerFolderIncludes= +#projectPerFolderExcludes= + +#wss.connectionTimeoutMinutes=60 + +# Change the below URL to your WhiteSource server. +# Use the 'WhiteSource Server URL' which can be retrieved +# from your 'Profile' page on the 'Server URLs' panel. +# Then, add the '/agent' path to it. +wss.url=https://saas.whitesourcesoftware.com/agent +#wss.url=https://app.whitesourcesoftware.com/agent +#wss.url=https://app-eu.whitesourcesoftware.com/agent + +############ +# Policies # +############ +checkPolicies=false +forceCheckAllDependencies=false +forceUpdate=false +forceUpdate.failBuildOnPolicyViolation=false +#updateInventory=false + +########### +# General # +########### +#offline=false +#updateType=APPEND +#ignoreSourceFiles=true +#scanComment= +#failErrorLevel=ALL +#requireKnownSha1=false + +#generateProjectDetailsJson=true +#generateScanReport=true +#scanReportTimeoutMinutes=10 +#scanReportFilenameFormat= + +#analyzeFrameworks=true +#analyzeFrameworksReference= + +#updateEmptyProject=false + +#log.files.level= +#log.files.maxFileSize= +#log.files.maxFilesCount= +#log.files.path= + +######################################## +# Package Manager Dependency resolvers # +######################################## +resolveAllDependencies=false +#excludeDependenciesFromNodes=.*commons-io.*,.*maven-model + +#npm.resolveDependencies=false +#npm.ignoreSourceFiles=false +#npm.includeDevDependencies=true +#npm.runPreStep=true +#npm.ignoreNpmLsErrors=true +#npm.ignoreScripts=true +#npm.yarnProject=true +#npm.accessToken= +#npm.identifyByNameAndVersion=true +#npm.yarn.frozenLockfile=true +#npm.resolveMainPackageJsonOnly=true +#npm.removeDuplicateDependencies=false +#npm.resolveAdditionalDependencies=true +#npm.failOnNpmLsErrors = +#npm.projectNameFromDependencyFile = true +#npm.resolveGlobalPackages=true +#npm.resolveLockFile=true + +#bower.resolveDependencies=false +#bower.ignoreSourceFiles=true +#bower.runPreStep=true + +#nuget.resolvePackagesConfigFiles=false +#nuget.resolveCsProjFiles=false +#nuget.resolveDependencies=false +#nuget.restoreDependencies=true +#nuget.preferredEnvironment= +#nuget.packagesDirectory= +#nuget.ignoreSourceFiles=false +#nuget.runPreStep=true +#nuget.resolveNuspecFiles=false +#nuget.resolveAssetsFiles=true + +#python.resolveDependencies=false +#python.ignoreSourceFiles=false +#python.ignorePipInstallErrors=true +#python.installVirtualenv=true +#python.resolveHierarchyTree=false +#python.requirementsFileIncludes=requirements.txt +#python.resolveSetupPyFiles=true +#python.runPipenvPreStep=true +#python.pipenvDevDependencies=true +#python.IgnorePipenvInstallErrors=true +#python.resolveGlobalPackages=true +#python.localPackagePathsToInstall=/path/to/local/dependency.egg, /path/to/local/dependency.zip +#python.resolvePipEditablePackages +#python.path=/path/to/python +#python.pipPath=/path/to/pip +#python.runPoetryPreStep=true +#python.includePoetryDevDependencies=true + +#maven.ignoredScopes=test provided +#maven.resolveDependencies=false +#maven.ignoreSourceFiles=true +#maven.aggregateModules=true +#maven.ignorePomModules=false +#maven.runPreStep=true +#maven.ignoreMvnTreeErrors=true +#maven.environmentPath= +#maven.m2RepositoryPath= +#maven.downloadMissingDependencies=false +#maven.additionalArguments= +#maven.projectNameFromDependencyFile=true + +resolveAllDependencies=false +archiveExtractionDepth=7 +followSymbolicLinks=true +gradle.resolveDependencies=true +gradle.aggregateModules=true +maven.resolveDependencies=true +maven.runPreStep=true +maven.aggregateModules=true +maven.ignoredScopes=None +npm.resolveDependencies=true +npm.runPreStep=true +npm.yarnProject=true +go.collectDependenciesAtRuntime=true +go.dependencyManager=modules +go.resolveDependencies=true + +#gradle.ignoredScopes= +#gradle.resolveDependencies=true +#gradle.runAssembleCommand=true +#gradle.runPreStep=true +#gradle.ignoreSourceFiles=true +#gradle.aggregateModules=true +#gradle.preferredEnvironment=wrapper +#gradle.localRepositoryPath= +#gradle.wrapperPath= +#gradle.downloadMissingDependencies=false +#gradle.additionalArguments= +#gradle.includedScopes= +#gradle.excludeModules= +#gradle.includeModules= +#gradle.includedConfigurations= +#gradle.ignoredConfigurations= + +#paket.resolveDependencies=false +#paket.ignoredGroups= +#paket.ignoreSourceFiles=false +#paket.runPreStep=true +#paket.exePath= + +#go.resolveDependencies=false +#go.collectDependenciesAtRuntime=true +#go.dependencyManager= +#go.ignoreSourceFiles=true +#go.glide.ignoreTestPackages=false +#go.gogradle.enableTaskAlias=true + +#ruby.resolveDependencies=false +#ruby.ignoreSourceFiles=false +#ruby.installMissingGems=true +#ruby.runBundleInstall=true +#ruby.overwriteGemFile=true + +#sbt.resolveDependencies=false +#sbt.ignoreSourceFiles=true +#sbt.aggregateModules=true +#sbt.runPreStep=true +#sbt.includedScopes= + +#php.resolveDependencies=false +#php.runPreStep=true +#php.includeDevDependencies=true + +#html.resolveDependencies=false + +#cocoapods.resolveDependencies=false +#cocoapods.runPreStep=true +#cocoapods.ignoreSourceFiles=false + +#hex.resolveDependencies=false +#hex.runPreStep=true +#hex.ignoreSourceFiles=false +#hex.aggregateModules=true + +#ant.resolveDependencies=false +#ant.pathIdIncludes=.* +#ant.external.parameters= + +#r.resolveDependencies=false +#r.runPreStep=true +#r.ignoreSourceFiles=false +#r.cranMirrorUrl= +#r.packageManager=None + +#cargo.resolveDependencies=false +#cargo.runPreStep=true +#cargo.ignoreSourceFiles=false + +#haskell.resolveDependencies=false +#haskell.runPreStep=true +#haskell.ignoreSourceFiles=false +#haskell.ignorePreStepErrors=true + +#ocaml.resolveDependencies=false +#ocaml.runPrepStep=true +#ocaml.ignoreSourceFiles=false +#ocaml.switchName= +#ocaml.ignoredScopes=none +#ocaml.aggregateModules=true + +#bazel.resolveDependencies=false +#bazel.runPrepStep=true + +########################################################################################### +# Includes/Excludes Glob patterns - Please use only one exclude line and one include line # +########################################################################################### +includes=**/*.c **/*.cc **/*.cp **/*.cpp **/*.cxx **/*.c++ **/*.h **/*.hpp **/*.hxx **/*.jar **/*.java + +#includes=**/*.m **/*.mm **/*.js **/*.php +#includes=**/*.jar +#includes=**/*.gem **/*.rb +#includes=**/*.dll **/*.cs **/*.nupkg +#includes=**/*.tgz **/*.deb **/*.gzip **/*.rpm **/*.tar.bz2 +#includes=**/*.zip **/*.tar.gz **/*.egg **/*.whl **/*.py + +#Exclude file extensions or specific directories by adding **/*. or **//** +excludes=**/*sources.jar **/*javadoc.jar + +case.sensitive.glob=false +followSymbolicLinks=true + +###################### +# Archive properties # +###################### +#archiveExtractionDepth=2 +#archiveIncludes=**/*.war **/*.ear +#archiveExcludes=**/*sources.jar + +############## +# SCAN MODES # +############## + +# Docker images +################ +#docker.scanImages=true +#docker.includes=.*.* +#docker.excludes= +#docker.pull.enable=true +#docker.pull.images=.*.* +#docker.pull.maxImages=10 +#docker.pull.tags=.*.* +#docker.pull.digest= +#docker.delete.force=true +#docker.login.sudo=false +#docker.projectNameFormat={repositoryNameAndTag|repositoryName|default} +#docker.scanTarFiles=true + +#docker.aws.enable=true +#docker.aws.registryIds= + +#docker.azure.enable=true +#docker.azure.userName= +#docker.azure.userPassword= +#docker.azure.registryNames= +#docker.azure.authenticationType=containerRegistry +#docker.azure.registryAuthenticationParameters=: : + +#docker.gcr.enable=true +#docker.gcr.account= +#docker.gcr.repositories= + +#docker.artifactory.enable=true +#docker.artifactory.url= +#docker.artifactory.pullUrl= +#docker.artifactory.userName= +#docker.artifactory.userPassword= +#docker.artifactory.repositoriesNames= +#docker.artifactory.dockerAccessMethod= + +#docker.hub.enabled=true +#docker.hub.userName= +#docker.hub.userPassword= +#docker.hub.organizationsNames= + +# Docker containers +#################### +#docker.scanContainers=true +#docker.containerIncludes=.*.* +#docker.containerExcludes= + +# Linux package manager settings +################################ +#scanPackageManager=true + +# Serverless settings +###################### +#serverless.provider= +#serverless.scanFunctions=true +#serverless.includes= +#serverless.excludes= +#serverless.region= +#serverless.maxFunctions=10 + +# Artifactory settings +######################## +#artifactory.enableScan=true +#artifactory.url= +#artifactory.accessToken= +#artifactory.repoKeys= +#artifactory.userName= +#artifactory.userPassword= + +################## +# Proxy settings # +################## +#proxy.host= +#proxy.port= +#proxy.user= +#proxy.pass= + +################ +# SCM settings # +################ +#scm.type= +#scm.user= +#scm.pass= +#scm.ppk= +#scm.url= +#scm.branch= +#scm.tag= +#scm.npmInstall= +#scm.npmInstallTimeoutMinutes= +#scm.repositoriesFile= From d020fdc0fc0dedfa13be4b5fa6efbe65acae5170 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 3 May 2021 16:27:59 -0400 Subject: [PATCH 2/5] Remove unnecessary folders Signed-off-by: Peter Zhu --- docker/.DS_Store | Bin 6148 -> 0 bytes docker/build-image.sh | 103 ----- .../opensearch-dashboards.al2.dockerfile | 92 ----- docker/dockerfiles/opensearch.al2.dockerfile | 97 ----- docker/opensearch-config/log4j2.properties | 9 - .../opensearch-docker-entrypoint.sh | 112 ------ .../opensearch-onetime-setup.sh | 50 --- docker/opensearch-config/opensearch.yml | 7 - .../performance-analyzer.properties | 47 --- ...opensearch-dashboards-docker-entrypoint.sh | 145 ------- .../opensearch.example.org.cert | 19 - .../opensearch.example.org.key | 27 -- .../opensearch_dashboards.yml | 39 -- opensearch/linux/opensearch-tar-install.sh | 73 ---- release-tools/scripts/setup_runners.sh | 264 ------------- .../vulnerability-scan/wss-scan.config | 3 - .../vulnerability-scan/wss-scan.sh | 105 ----- .../wss-unified-agent.config | 363 ------------------ 18 files changed, 1555 deletions(-) delete mode 100644 docker/.DS_Store delete mode 100755 docker/build-image.sh delete mode 100644 docker/dockerfiles/opensearch-dashboards.al2.dockerfile delete mode 100644 docker/dockerfiles/opensearch.al2.dockerfile delete mode 100644 docker/opensearch-config/log4j2.properties delete mode 100755 docker/opensearch-config/opensearch-docker-entrypoint.sh delete mode 100755 docker/opensearch-config/opensearch-onetime-setup.sh delete mode 100644 docker/opensearch-config/opensearch.yml delete mode 100644 docker/opensearch-config/performance-analyzer.properties delete mode 100755 docker/opensearch-dashboards-config/opensearch-dashboards-docker-entrypoint.sh delete mode 100644 docker/opensearch-dashboards-config/opensearch.example.org.cert delete mode 100644 docker/opensearch-dashboards-config/opensearch.example.org.key delete mode 100644 docker/opensearch-dashboards-config/opensearch_dashboards.yml delete mode 100755 opensearch/linux/opensearch-tar-install.sh delete mode 100755 release-tools/scripts/setup_runners.sh delete mode 100644 standalone-tools/vulnerability-scan/wss-scan.config delete mode 100755 standalone-tools/vulnerability-scan/wss-scan.sh delete mode 100644 standalone-tools/vulnerability-scan/wss-unified-agent.config diff --git a/docker/.DS_Store b/docker/.DS_Store deleted file mode 100644 index 02ef0a302a9deb64100bdb27aac46745c8d3a172..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK!Ab)$5S`SjTSV+Z&|@y%dT6P%Ct<02@FuM2L8b1xt1hk^>257rEBhPzM}C37 z<4lrPr1m6;l$kPllgUg%UP@*I0HV_m8vs=Ra8LNSG8Yi8SlJyV@Jx2sRxPllG zh^C_1@gEtWvs-}>6kNhBp4!*>Wx*i34-yq%#Cvd`#93aie~8jTd2wmkso-zbIrnbV zz{~x-pLP7+DfJHh{3fvWky6RT_JiX%8g#4cyDH87IE{KbISwNXIXj8dPz^e&pN5&9 z$1?-8PNiGj91dIUW=%Hhtx-)5+wFQyZZ}(_QN`KVYU~_b+&w-GpGPmRhGT($M=fg> z$MA;6jO{*ny);qjHAalH$61Wb05iZ0%ozjrnsZj>Y+t+`W`G&^Jp*( ztD|4)a1gFRZkYjQ;5!2grdy@@zd!r^|2>I&%m6bmR}6^qp10S*E!o;SwK=M_67>R= pgyIT=lN2;`DaKeT#ha*F&@ahAbS)MJ(SyPt0-6SHn1MfK;1iRKRLlSX diff --git a/docker/build-image.sh b/docker/build-image.sh deleted file mode 100755 index 3195cc1a3d..0000000000 --- a/docker/build-image.sh +++ /dev/null @@ -1,103 +0,0 @@ -#!/bin/bash - -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. - - -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - - -set -e - -function usage() { - echo "" - echo "This script is used to build the OpenSearch Docker image. It prepares the files required by the Dockerfile in a temporary directory, then builds and tags the Docker image." - echo "--------------------------------------------------------------------------" - echo "Usage: $0 [args]" - echo "" - echo "Required arguments:" - echo -e "-v VERSION\tSpecify the OpenSearch version number that you are building, e.g. '1.0.0' or '1.0.0-beta1'. This will be used to label the Docker image. If you do not use the '-o' option then this tool will download a public OPENSEARCH release matching this version." - echo -e "-f DOCKERFILE\tSpecify the dockerfile full path, e.g. dockerfile/opensearch.al2.dockerfile." - echo -e "-p PRODUCT\tSpecify the product, e.g. opensearch or opensearch-dashboards, make sure this is the name of your .tgz defined in dockerfile." - echo "" - echo "Optional arguments:" - echo -e "-o FILENAME\tSpecify a local OPENSEARCH tarball. You still need to specify the version - this tool does not attempt to parse the filename." - echo -e "-h\t\tPrint this message." - echo "--------------------------------------------------------------------------" -} - -while getopts ":ho:v:f:p:" arg; do - case $arg in - h) - usage - exit 1 - ;; - o) - TARBALL=`realpath $OPTARG` - ;; - v) - VERSION=$OPTARG - ;; - f) - DOCKERFILE=$OPTARG - ;; - p) - PRODUCT=$OPTARG - ;; - :) - echo "-${OPTARG} requires an argument" - usage - exit 1 - ;; - ?) - echo "Invalid option: -${arg}" - exit 1 - ;; - esac -done - -if [ -z "$VERSION" ] || [ -z "$DOCKERFILE" ] || [ -z "$PRODUCT" ]; then - echo "You must specify '-v VERSION', '-f DOCKERFILE', '-p PRODUCT'" - usage - exit 1 -fi - -echo $DOCKERFILE - -DIR=`mktemp -d` - -echo "Creating Docker workspace in $DIR" -trap '{ echo Removing Docker workspace in "$DIR"; rm -rf -- "$DIR"; }' TERM INT EXIT - -if [ -z "$TARBALL" ]; then - # No tarball file specified so download one - URL="https://artifacts.opensearch.org/releases/bundle/${PRODUCT}/${VERSION}/${PRODUCT}-${VERSION}-linux-x64.tar.gz" - echo "Downloading ${PRODUCT} version ${VERSION} from ${URL}" - curl -f $URL -o $DIR/$PRODUCT.tgz || exit 1 - ls -l $DIR -else - cp -v $TARBALL $DIR/$PRODUCT.tgz -fi - -cp -v ${PRODUCT}-config/* $DIR/ - -docker build --build-arg VERSION=$VERSION --build-arg BUILD_DATE=`date -u +%Y-%m-%dT%H:%M:%SZ` -f $DOCKERFILE $DIR -t opensearchproject/$PRODUCT:$VERSION - -rm -rf $DIR diff --git a/docker/dockerfiles/opensearch-dashboards.al2.dockerfile b/docker/dockerfiles/opensearch-dashboards.al2.dockerfile deleted file mode 100644 index b70b90669c..0000000000 --- a/docker/dockerfiles/opensearch-dashboards.al2.dockerfile +++ /dev/null @@ -1,92 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. - - -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - - -# This dockerfile generates an AmazonLinux-based image containing an OpenSearch-Dashboards installation. -# It assumes that the working directory contains four files: an OpenSearch-Dashboards tarball (opensearch-dashboards.tgz), opensearch_dashboards.yml, opensearch-dashboards-docker-entrypoint.sh, and example certs. -# Build arguments: -# VERSION: Required. Used to label the image. -# BUILD_DATE: Required. Used to label the image. Should be in the form 'yyyy-mm-ddThh:mm:ssZ', i.e. a date-time from https://tools.ietf.org/html/rfc3339. The timestamp must be in UTC. -# UID: Optional. Specify the opensearch-dashboards userid. Defaults to 1000. -# GID: Optional. Specify the opensearch-dashboards groupid. Defaults to 1000. -# OPENSEARCH_DASHBOARDS_HOME: Optional. Specify the opensearch-dashboards root directory. Defaults to /usr/share/opensearch-dashboards. - - -FROM amazonlinux:2 AS linux_x64_staging - -ARG UID=1000 -ARG GID=1000 -ARG OPENSEARCH_DASHBOARDS_HOME=/usr/share/opensearch-dashboards - -# Update packages -# Install the tools we need: tar and gzip to unpack the OpenSearch tarball, and shadow-utils to give us `groupadd` and `useradd`. -RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all - -# Create an opensearch-dashboards user, group, and directory -RUN groupadd -g $GID opensearch-dashboards && \ - adduser -u $UID -g $GID -d $OPENSEARCH_DASHBOARDS_HOME opensearch-dashboards && \ - mkdir /tmp/opensearch-dashboards - -# Prepare working directory -COPY opensearch-dashboards.tgz /tmp/opensearch-dashboards/opensearch-dashboards.tgz -RUN tar -xzf /tmp/opensearch-dashboards/opensearch-dashboards.tgz -C $OPENSEARCH_DASHBOARDS_HOME --strip-components=1 && rm -rf /temp/opensearch-dashboards -COPY opensearch-dashboards-docker-entrypoint.sh $OPENSEARCH_DASHBOARDS_HOME/ -COPY opensearch_dashboards.yml opensearch.example.org.* $OPENSEARCH_DASHBOARDS_HOME/config/ - -# Copy working directory to the actual release docker images -FROM amazonlinux:2 - -ARG UID=1000 -ARG GID=1000 -ARG OPENSEARCH_DASHBOARDS_HOME=/usr/share/opensearch-dashboards - -COPY --from=linux_x64_staging $OPENSEARCH_DASHBOARDS_HOME $OPENSEARCH_DASHBOARDS_HOME - -# Setup OpenSearch-dashboards -WORKDIR $OPENSEARCH_DASHBOARDS_HOME -RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all -RUN groupadd -g $GID opensearch-dashboards && \ - adduser -u $UID -g $GID -d $OPENSEARCH_DASHBOARDS_HOME opensearch-dashboards -RUN chown -R $UID:$GID $OPENSEARCH_DASHBOARDS_HOME - -# Change user -USER $UID - -# Expose port -EXPOSE 5601 - -ARG VERSION -ARG BUILD_DATE - -# Label -LABEL org.label-schema.schema-version="1.0" \ - org.label-schema.name="opensearch-dashboards" \ - org.label-schema.version="$VERSION" \ - org.label-schema.url="https://opensearch.org" \ - org.label-schema.vcs-url="https://github.com/opensearch-project/OpenSearch-Dashboards" \ - org.label-schema.license="Apache-2.0" \ - org.label-schema.vendor="Amazon" \ - org.label-schema.build-date="$BUILD_DATE" - -# CMD to run -CMD ["./opensearch-dashboards-docker-entrypoint.sh"] diff --git a/docker/dockerfiles/opensearch.al2.dockerfile b/docker/dockerfiles/opensearch.al2.dockerfile deleted file mode 100644 index 747fc995d5..0000000000 --- a/docker/dockerfiles/opensearch.al2.dockerfile +++ /dev/null @@ -1,97 +0,0 @@ -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. - - -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - - -# This dockerfile generates an AmazonLinux-based image containing an OpenSearch installation. -# It assumes that the working directory contains four files: an OpenSearch tarball (opensearch.tgz), log4j2.properties, opensearch.yml, opensearch-docker-entrypoint.sh, opensearch-onetime-setup.sh. -# Build arguments: -# VERSION: Required. Used to label the image. -# BUILD_DATE: Required. Used to label the image. Should be in the form 'yyyy-mm-ddThh:mm:ssZ', i.e. a date-time from https://tools.ietf.org/html/rfc3339. The timestamp must be in UTC. -# UID: Optional. Specify the opensearch userid. Defaults to 1000. -# GID: Optional. Specify the opensearch groupid. Defaults to 1000. -# OPENSEARCH_HOME: Optional. Specify the opensearch root directory. Defaults to /usr/share/opensearch. -# PA_NAME: Optional. Specify the opensearch performance analyzer plugin folder name. Defaults to opensearch-performance-analyzer. - - -FROM amazonlinux:2 AS linux_x64_staging - -ARG UID=1000 -ARG GID=1000 -ARG OPENSEARCH_HOME=/usr/share/opensearch -ARG PA_NAME=opensearch-performance-analyzer - -# Update packages -# Install the tools we need: tar and gzip to unpack the OpenSearch tarball, and shadow-utils to give us `groupadd` and `useradd`. -RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all - -# Create an opensearch user, group, and directory -RUN groupadd -g $GID opensearch && \ - adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch && \ - mkdir /tmp/opensearch - -# Prepare working directory -COPY opensearch.tgz /tmp/opensearch/opensearch.tgz -RUN tar -xzf /tmp/opensearch/opensearch.tgz -C $OPENSEARCH_HOME --strip-components=1 && rm -rf /temp/opensearch -COPY opensearch-docker-entrypoint.sh opensearch-onetime-setup.sh $OPENSEARCH_HOME/ -COPY log4j2.properties opensearch.yml $OPENSEARCH_HOME/config/ -COPY performance-analyzer.properties $OPENSEARCH_HOME/plugins/$PA_NAME/pa_config/ - -# Copy working directory to the actual release docker images -FROM amazonlinux:2 - -ARG UID=1000 -ARG GID=1000 -ARG OPENSEARCH_HOME=/usr/share/opensearch -ARG PA_NAME=opensearch-performance-analyzer - -COPY --from=linux_x64_staging $OPENSEARCH_HOME $OPENSEARCH_HOME - -# Setup OpenSearch -WORKDIR $OPENSEARCH_HOME -RUN yum update -y && yum install -y tar gzip shadow-utils && yum clean all -RUN groupadd -g $GID opensearch && \ - adduser -u $UID -g $GID -d $OPENSEARCH_HOME opensearch -RUN ./opensearch-onetime-setup.sh && \ - chown -R $UID:$GID $OPENSEARCH_HOME - -# Change user -USER $UID - -# Expose ports for the opensearch service (9200 for HTTP and 9300 for internal transport) and performance analyzer (9600 for the agent and 9650 for the root cause analysis component) -EXPOSE 9200 9300 9600 9650 - -ARG VERSION -ARG BUILD_DATE - -# Label -LABEL org.label-schema.schema-version="1.0" \ - org.label-schema.name="opensearch" \ - org.label-schema.version="$VERSION" \ - org.label-schema.url="https://opensearch.org" \ - org.label-schema.vcs-url="https://github.com/OpenSearch" \ - org.label-schema.license="Apache-2.0" \ - org.label-schema.vendor="Amazon" \ - org.label-schema.build-date="$BUILD_DATE" - -# CMD to run -CMD ["./opensearch-docker-entrypoint.sh"] diff --git a/docker/opensearch-config/log4j2.properties b/docker/opensearch-config/log4j2.properties deleted file mode 100644 index 9ad290ad82..0000000000 --- a/docker/opensearch-config/log4j2.properties +++ /dev/null @@ -1,9 +0,0 @@ -status = error - -appender.console.type = Console -appender.console.name = console -appender.console.layout.type = PatternLayout -appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker %m%n - -rootLogger.level = info -rootLogger.appenderRef.console.ref = console diff --git a/docker/opensearch-config/opensearch-docker-entrypoint.sh b/docker/opensearch-config/opensearch-docker-entrypoint.sh deleted file mode 100755 index 305cf5c8da..0000000000 --- a/docker/opensearch-config/opensearch-docker-entrypoint.sh +++ /dev/null @@ -1,112 +0,0 @@ -#!/bin/bash - -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. - - -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - -# This script specify the entrypoint startup actions for opensearch -# It will start both opensearch and performance analyzer plugin cli -# If either process failed, the entire docker container will be removed -# in favor of a newly started container - -# Files created by OpenSearch should always be group writable too -umask 0002 - -if [[ "$(id -u)" == "0" ]]; then - echo "OpenSearch cannot run as root. Please start your container as another user." - exit 1 -fi - -# Parse Docker env vars to customize OpenSearch -# -# e.g. Setting the env var cluster.name=testcluster -# -# will cause OpenSearch to be invoked with -Ecluster.name=testcluster - -declare -a opensearch_opts - -while IFS='=' read -r envvar_key envvar_value -do - # OpenSearch settings need to have at least two dot separated lowercase - # words, e.g. `cluster.name`, except for `processors` which we handle - # specially - if [[ "$envvar_key" =~ ^[a-z0-9_]+\.[a-z0-9_]+ || "$envvar_key" == "processors" ]]; then - if [[ ! -z $envvar_value ]]; then - opensearch_opt="-E${envvar_key}=${envvar_value}" - opensearch_opts+=("${opensearch_opt}") - fi - fi -done < <(env) - -# The virtual file /proc/self/cgroup should list the current cgroup -# membership. For each hierarchy, you can follow the cgroup path from -# this file to the cgroup filesystem (usually /sys/fs/cgroup/) and -# introspect the statistics for the cgroup for the given -# hierarchy. Alas, Docker breaks this by mounting the container -# statistics at the root while leaving the cgroup paths as the actual -# paths. Therefore, OpenSearch provides a mechanism to override -# reading the cgroup path from /proc/self/cgroup and instead uses the -# cgroup path defined the JVM system property -# es.cgroups.hierarchy.override. Therefore, we set this value here so -# that cgroup statistics are available for the container this process -# will run in. -export OPENSEARCH_JAVA_OPTS="-Dopensearch.cgroups.hierarchy.override=/ $OPENSEARCH_JAVA_OPTS" - - -# Start up the opensearch and performance analyzer agent processes. -# When either of them halts, this script exits, or we receive a SIGTERM or SIGINT signal then we want to kill both these processes. - -function terminateProcesses { - if kill -0 $OPENSEARCH_PID >& /dev/null; then - echo "Killing opensearch process $OPENSEARCH_PID" - kill -TERM $OPENSEARCH_PID - wait $OPENSEARCH_PID - fi - if kill -0 $PA_PID >& /dev/null; then - echo "Killing performance analyzer process $PA_PID" - kill -TERM $PA_PID - wait $PA_PID - fi -} - -# Enable job control so we receive SIGCHLD when a child process terminates -set -m - -# Make sure we terminate the child processes in the event of us received TERM (e.g. "docker container stop"), INT (e.g. ctrl-C), EXIT (this script terminates for an unexpected reason), or CHLD (one of the processes terminated unexpectedly) -trap terminateProcesses TERM INT EXIT CHLD - -# Export OpenSearch Home -export OPENSEARCH_HOME=/usr/share/opensearch - -# Start elasticsearch -$OPENSEARCH_HOME/bin/opensearch "${opensearch_opts[@]}" & -OPENSEARCH_PID=$! - -# Start performance analyzer agent -$OPENSEARCH_HOME/bin/performance-analyzer-agent-cli & -PA_PID=$! - -# Wait for the child processes to terminate -wait $OPENSEARCH_PID -echo "Elasticsearch exited with code $?" -wait $PA_PID -echo "Performance analyzer exited with code $?" diff --git a/docker/opensearch-config/opensearch-onetime-setup.sh b/docker/opensearch-config/opensearch-onetime-setup.sh deleted file mode 100755 index 3f07ede5ea..0000000000 --- a/docker/opensearch-config/opensearch-onetime-setup.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. - - -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - - -# This script performs one-time setup for the OpenSearch tarball distribution. -# It installs a demo security config and sets up the performance analyzer - -OPENSEARCH_HOME=`dirname $(realpath $0)`; cd $OPENSEARCH_HOME - -##Security Plugin -SECURITY_PLUGIN="opensearch-security" -bash $OPENSEARCH_HOME/plugins/$SECURITY_PLUGIN/tools/install_demo_configuration.sh -y -i -s - -##Perf Plugin -PA_PLUGIN="opensearch-performance-analyzer" -chmod 755 $OPENSEARCH_HOME/plugins/$PA_PLUGIN/pa_bin/performance-analyzer-agent -chmod -R 755 /dev/shm -chmod 755 $OPENSEARCH_HOME/bin/performance-analyzer-agent-cli - -if ! grep -q '## OpenDistro Performance Analyzer' $OPENSEARCH_HOME/config/jvm.options; then - CLK_TCK=`/usr/bin/getconf CLK_TCK` - echo >> $OPENSEARCH_HOME/config/jvm.options - echo '## OpenDistro Performance Analyzer' >> $OPENSEARCH_HOME/config/jvm.options - echo "-Dclk.tck=$CLK_TCK" >> $OPENSEARCH_HOME/config/jvm.options - echo "-Djdk.attach.allowAttachSelf=true" >> $OPENSEARCH_HOME/config/jvm.options - echo "-Djava.security.policy=$OPENSEARCH_HOME/plugins/$PA_PLUGIN/pa_config/opensearch_security.policy" >> $OPENSEARCH_HOME/config/jvm.options -fi - diff --git a/docker/opensearch-config/opensearch.yml b/docker/opensearch-config/opensearch.yml deleted file mode 100644 index 748b2235e5..0000000000 --- a/docker/opensearch-config/opensearch.yml +++ /dev/null @@ -1,7 +0,0 @@ -cluster.name: docker-cluster - -# Bind to all interfaces because we don't know what IP address Docker will assign to us. -network.host: 0.0.0.0 - -# Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again. -discovery.type: single-node diff --git a/docker/opensearch-config/performance-analyzer.properties b/docker/opensearch-config/performance-analyzer.properties deleted file mode 100644 index 72235cc9a8..0000000000 --- a/docker/opensearch-config/performance-analyzer.properties +++ /dev/null @@ -1,47 +0,0 @@ -# ======================== OpenSearch performance analyzer plugin config ========================= - -# NOTE: this is an example for Linux. Please modify the config accordingly if you are using it under other OS. - -# Metrics data location -metrics-location = /dev/shm/performanceanalyzer/ - -# Metrics deletion interval (minutes) for metrics data. -# Interval should be between 1 to 60. -metrics-deletion-interval = 1 - -# If set to true, the system cleans up the files behind it. So at any point, we should expect only 2 -# metrics-db-file-prefix-path files. If set to false, no files are cleaned up. This can be useful, if you are archiving -# the files and wouldn't like for them to be cleaned up. -cleanup-metrics-db-files = true - -# WebService exposed by App's port -webservice-listener-port = 9600 - -# Port for RPC Communication -rpc-port = 9650 - -# Metric DB File Prefix Path location -metrics-db-file-prefix-path = /tmp/metricsdb_ - -https-enabled = false - -# Setup the correct path for server certificates -certificate-file-path = none -private-key-file-path = none -#trusted-cas-file-path = none - -# Setup the correct path for client certificates (by default, the client will just use the server certificates) -#client-certificate-file-path = specify_path -#client-private-key-file-path = specify_path -#client-trusted-cas-file-path = specify_path - -# WebService bind host; default only to local interface -webservice-bind-host = 0.0.0.0 - -# Plugin Stats Metadata file name, expected to be in the same location -plugin-stats-metadata = plugin-stats-metadata - -# Agent Stats Metadata file name, expected to be in the same location -agent-stats-metadata = agent-stats-metadata - - diff --git a/docker/opensearch-dashboards-config/opensearch-dashboards-docker-entrypoint.sh b/docker/opensearch-dashboards-config/opensearch-dashboards-docker-entrypoint.sh deleted file mode 100755 index ac5ad70b4c..0000000000 --- a/docker/opensearch-dashboards-config/opensearch-dashboards-docker-entrypoint.sh +++ /dev/null @@ -1,145 +0,0 @@ -#!/bin/bash - -# Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - -# -# Run Kibana, using environment variables to set longopts defining Kibana's -# configuration. -# -# eg. Setting the environment variable: -# -# ELASTICSEARCH_STARTUPTIMEOUT=60 -# -# will cause Kibana to be invoked with: -# -# --elasticsearch.startupTimeout=60 - -opensearch_dashboards_vars=( - console.enabled - console.proxyConfig - console.proxyFilter - elasticsearch.customHeaders - elasticsearch.logQueries - elasticsearch.password - elasticsearch.pingTimeout - elasticsearch.preserveHost - elasticsearch.requestHeadersWhitelist - elasticsearch.requestTimeout - elasticsearch.shardTimeout - elasticsearch.ssl.ca - elasticsearch.ssl.cert - elasticsearch.ssl.certificate - elasticsearch.ssl.certificateAuthorities - elasticsearch.ssl.key - elasticsearch.ssl.keyPassphrase - elasticsearch.ssl.verificationMode - elasticsearch.ssl.verify - elasticsearch.startupTimeout - elasticsearch.tribe.customHeaders - elasticsearch.tribe.password - elasticsearch.tribe.pingTimeout - elasticsearch.tribe.requestHeadersWhitelist - elasticsearch.tribe.requestTimeout - elasticsearch.tribe.ssl.ca - elasticsearch.tribe.ssl.cert - elasticsearch.tribe.ssl.certificate - elasticsearch.tribe.ssl.certificateAuthorities - elasticsearch.tribe.ssl.key - elasticsearch.tribe.ssl.keyPassphrase - elasticsearch.tribe.ssl.verificationMode - elasticsearch.tribe.ssl.verify - elasticsearch.tribe.url - elasticsearch.tribe.username - elasticsearch.hosts - kibana.defaultAppId - kibana.index - logging.dest - logging.quiet - logging.silent - logging.useUTC - logging.verbose - map.includeElasticMapsService - ops.interval - path.data - pid.file - regionmap - regionmap.includeElasticMapsService - server.basePath - server.customResponseHeaders - server.defaultRoute - server.host - server.maxPayloadBytes - server.name - server.port - server.rewriteBasePath - server.ssl.cert - server.ssl.certificate - server.ssl.certificateAuthorities - server.ssl.cipherSuites - server.ssl.clientAuthentication - server.customResponseHeaders - server.ssl.enabled - server.ssl.key - server.ssl.keyPassphrase - server.ssl.redirectHttpFromPort - server.ssl.supportedProtocols - server.xsrf.whitelist - status.allowAnonymous - status.v6ApiFormat - tilemap.options.attribution - tilemap.options.maxZoom - tilemap.options.minZoom - tilemap.options.subdomains - tilemap.url - timelion.enabled - vega.enableExternalUrls - opensearch_security.multitenancy.enabled - opensearch_security.multitenancy.tenants.preferred - opensearch_security.readonly_mode.roles -) - -longopts='' -for opensearch_dashboards_var in ${opensearch_dashboards_vars[*]}; do - # 'opensearch.hosts' -> 'OPENSEARCH_URL' - env_var=$(echo ${opensearch_dashboards_var^^} | tr . _) - - # Indirectly lookup env var values via the name of the var. - # REF: http://tldp.org/LDP/abs/html/bashver2.html#EX78 - value=${!env_var} - if [[ -n $value ]]; then - longopt="--${opensearch_dashboards_var}=${value}" - longopts+=" ${longopt}" - fi -done - -# Files created at run-time should be group-writable, for Openshift's sake. -umask 0002 - -# TO DO: -# Confirm with Mihir if this is necessary - -# The virtual file /proc/self/cgroup should list the current cgroup -# membership. For each hierarchy, you can follow the cgroup path from -# this file to the cgroup filesystem (usually /sys/fs/cgroup/) and -# introspect the statistics for the cgroup for the given -# hierarchy. Alas, Docker breaks this by mounting the container -# statistics at the root while leaving the cgroup paths as the actual -# paths. Therefore, OpenSearch-Dashboards provides a mechanism to override -# reading the cgroup path from /proc/self/cgroup and instead uses the -# cgroup path defined the configuration properties -# cpu.cgroup.path.override and cpuacct.cgroup.path.override. -# Therefore, we set this value here so that cgroup statistics are -# available for the container this process will run in. - -exec /usr/share/opensearch-dashboards/bin/opensearch-dashboards --cpu.cgroup.path.override=/ --cpuacct.cgroup.path.override=/ ${longopts} "$@" diff --git a/docker/opensearch-dashboards-config/opensearch.example.org.cert b/docker/opensearch-dashboards-config/opensearch.example.org.cert deleted file mode 100644 index 9540047b98..0000000000 --- a/docker/opensearch-dashboards-config/opensearch.example.org.cert +++ /dev/null @@ -1,19 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDJTCCAg2gAwIBAgIJAKO3AgiyHbReMA0GCSqGSIb3DQEBBQUAMCkxJzAlBgNV -BAMMHm9wZW5kaXN0cm9mb3JlbGFzdGljc2VhcmNoLm9yZzAeFw0xOTAyMTUwNzI5 -MjNaFw0yOTAyMTIwNzI5MjNaMCkxJzAlBgNVBAMMHm9wZW5kaXN0cm9mb3JlbGFz -dGljc2VhcmNoLm9yZzCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALPO -w4iMMWj7wODNeAR+t9fFoFkhWmo7NiV9qteUtOTnOLBFG6VWVh2kQev1Ph2dF0uw -Dg632Pk3Tl1er+eKCoG+YzwkJqVZ0CJlxZdpN5jCpliCYxIaGUHO0I+Kdhzg1IUk -1Q2+8aeVN7ddiYZC13T/b5Kut8lr13O8OTqeBX2zRxuzX+jUNAZjAsCAdsv2jRrO -94883OyDzF6UXvETJcW5uF4j7NK9N2nfAplMy6Z7v/0kA89/ecuAb8wmf7MOul1h -sHyNd4aJwIGhKW/qBLJG+GHB5BiO6OaMJMzr5kzSV9we5r/plas5viAl/jWsYECS -rvGjUqBEOKOFy6RS2ZECAwEAAaNQME4wHQYDVR0OBBYEFI05BOGEHtUhrZmkIL3W -6KpJLE71MB8GA1UdIwQYMBaAFI05BOGEHtUhrZmkIL3W6KpJLE71MAwGA1UdEwQF -MAMBAf8wDQYJKoZIhvcNAQEFBQADggEBACvAn1fRvkJamVKdmsou+YTmxu5YhDva -MepBVT6yeqctcgQgUfaneuBrl/+9zmxkjykZ1uimSS2IM1FxIXuB8rX+iAcgFyIs -XXquOhJlaXs/kN0KBGUSRkTMQSGIaTh26U7GnCQrdaIZDtKp7CRWjNebeDucNOc+ -aXOqiNFbVePNe8+EZIDb0FmV4IiHuLcqCwzZvgesl3EUPuMsnrxeNKIw83ecpjqK -YRwWWhvY57asVbOZjn+M57bJSpzO9bO70uib0ItWs19aWqYP1vEOuNHEDbr2EVel -dejL2fYF06uiw179PSYnvDQohWdiqm8dM0Nx+QtJcbOR6BiBh8aQTjw= ------END CERTIFICATE----- diff --git a/docker/opensearch-dashboards-config/opensearch.example.org.key b/docker/opensearch-dashboards-config/opensearch.example.org.key deleted file mode 100644 index e549011659..0000000000 --- a/docker/opensearch-dashboards-config/opensearch.example.org.key +++ /dev/null @@ -1,27 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIEowIBAAKCAQEAs87DiIwxaPvA4M14BH6318WgWSFaajs2JX2q15S05Oc4sEUb -pVZWHaRB6/U+HZ0XS7AODrfY+TdOXV6v54oKgb5jPCQmpVnQImXFl2k3mMKmWIJj -EhoZQc7Qj4p2HODUhSTVDb7xp5U3t12JhkLXdP9vkq63yWvXc7w5Op4FfbNHG7Nf -6NQ0BmMCwIB2y/aNGs73jzzc7IPMXpRe8RMlxbm4XiPs0r03ad8CmUzLpnu//SQD -z395y4BvzCZ/sw66XWGwfI13honAgaEpb+oEskb4YcHkGI7o5owkzOvmTNJX3B7m -v+mVqzm+ICX+NaxgQJKu8aNSoEQ4o4XLpFLZkQIDAQABAoIBAEtPKXRyi2tWLlW5 -mTAAxw1Xl65dGtptjGMtfDWo8g8BjSu+9jJMs+rcVkP2xppupkgsJ+cgN39CG89d -jj1jAr2OuU+CMLGGqXcOuzCMgB26QSPHEKmPRs7aSV4ldwznWfueDqIgPGSdSeXC -Boy2SDmXluqPxShWw7mUtz+G2fPoQ+TMcH3puL3ooeV2iy25ZciLG3SFFQ7jTiJP -l6bNvnszVa7fRdapF25znC/XsfaW7XsXZwbV+NeWtO7CEIec/RExSy9N/abbmhWY -BPidBT9AQG4MoN10YunS7iwbfpXCNW0HKlFf77Mr5FrCZ0RE/Mg5JJrFF3Bt+sNT -v6/2+wUCgYEA3mEtewQCn40aIWYTTeXqtRCnpbt4y602B2PbY7OI/9N6xafATVnK -+bwcj9rKQRmUBbrvNGCY/GQJPqHRFjIRqK0y2QCavuR/980z96ojUXwsHG4eSmHY -wX0Rs+JtHVJDDH13hRWj/5Zxa+0ln2olBpHkp+o/VwiyYNXeqmw4Lq8CgYEAzv3p -kmOqNQpQ2y2jEdKYanIibsB27slb4UxBpCzEYSj3LF3CysaGF17GJRRCASH9Dm3Y -dZSgJzuBVZSgYl9HNSB1wY1gpHYeLm0F0yaE1D4tSoXGD+8pbLbl6LjwHOUtH4Yx -NCDwOv+pe6+tyO6v04xcAOSWWkcsHkesrcehi78CgYEAztNA8o3LRst8Rx3hzGyP -O7HuyoSvF3fUrVDhZjw/qfft0tFvWEGwN08OnXjW7W81rZKsEWHaR7teocLsbSFK -LUBQ72TRh8OLhVR0Bd9Wk93SROrh/zo719FT2qM36rJsq7wxojR7YCZy/PcGK9sT -7qVYLhf+qxrqlIe1XXnjUv8CgYBhglm63Xp8HAz1A22zYqrsUjIITGB6op7G5p5x -kak9QAgYfV7bGWRns/zR0NY7JGDKoFJ9KXO8p5qIz0gyr0GjNFRkxo4FS6NyKWe6 -DYY7WcMKPTSNoxO3o/1wgVaTth5ooFWjgfTJRlD39E2Av3hSxuc8gHKXurl81jTx -dH6UjwKBgEA5JLPaICkkY4ppcwdnf7zFFi22Rq5e2AJQSJcHOgE0K8/jJqbv+VOy -ugVjXRpq6VdNjGD3aLToe/SnRLUDmq+ZJqV+BoZSv3DJCsUkc4O9tS/E6Tn4A1di -loi+vT/GFEVVE3MaB42XGaaC82VEgwdqE3hDqNM/F2i3DfyoKD6V ------END RSA PRIVATE KEY----- \ No newline at end of file diff --git a/docker/opensearch-dashboards-config/opensearch_dashboards.yml b/docker/opensearch-dashboards-config/opensearch_dashboards.yml deleted file mode 100644 index f34b782048..0000000000 --- a/docker/opensearch-dashboards-config/opensearch_dashboards.yml +++ /dev/null @@ -1,39 +0,0 @@ ---- -# SPDX-License-Identifier: Apache-2.0 -# -# The OpenSearch Contributors require contributions made to -# this file be licensed under the Apache-2.0 license or a -# compatible open source license. -# -# Modifications Copyright OpenSearch Contributors. See -# GitHub history for details. - - -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - -# Description: -# Default configuration for OpenSearch Dashboards - -opensearch.hosts: ["http://localhost:9200"] -opensearch.ssl.verificationMode: full -opensearch.username: "kibanaserver" -opensearch.password: "kibanaserver" -opensearch.requestHeadersWhitelist: [ authorization ] - -opensearch_security.multitenancy.enabled: true -opensearch_security.multitenancy.tenants.preferred: ["Private", "Global"] -opensearch_security.readonly_mode.roles: ["kibana_read_only"] -# Use this setting if you are running kibana without https -opensearch_security.cookie.secure: false - diff --git a/opensearch/linux/opensearch-tar-install.sh b/opensearch/linux/opensearch-tar-install.sh deleted file mode 100755 index d51781facf..0000000000 --- a/opensearch/linux/opensearch-tar-install.sh +++ /dev/null @@ -1,73 +0,0 @@ -#!/bin/bash - -# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). -# You may not use this file except in compliance with the License. -# A copy of the License is located at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# or in the "license" file accompanying this file. This file is distributed -# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either -# express or implied. See the License for the specific language governing -# permissions and limitations under the License. - -OPENSEARCH_HOME=`dirname $(realpath $0)`; cd $OPENSEARCH_HOME -KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/knnlib -##Security Plugin -bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s - -##Perf Plugin -chmod 755 $OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_bin/performance-analyzer-agent -chmod -R 755 /dev/shm -chmod 755 $OPENSEARCH_HOME/bin/performance-analyzer-agent-cli -echo "done security" -PA_AGENT_JAVA_OPTS="-Dlog4j.configurationFile=$OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_config/log4j2.xml \ - -Xms64M -Xmx64M -XX:+UseSerialGC -XX:CICompilerCount=1 -XX:-TieredCompilation -XX:InitialCodeCacheSize=4096 \ - -XX:InitialBootClassLoaderMetaspaceSize=30720 -XX:MaxRAM=400m" - -OPENSEARCH_MAIN_CLASS="com.amazon.opendistro.opensearch.performanceanalyzer.PerformanceAnalyzerApp" \ -OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=plugins/opensearch-performance-analyzer \ -OPENSEARCH_JAVA_OPTS=$PA_AGENT_JAVA_OPTS - -if ! grep -q '## OpenSearch Performance Analyzer' $OPENSEARCH_HOME/config/jvm.options; then - CLK_TCK=`/usr/bin/getconf CLK_TCK` - echo >> $OPENSEARCH_HOME/config/jvm.options - echo '## OpenSearch Performance Analyzer' >> $OPENSEARCH_HOME/config/jvm.options - echo "-Dclk.tck=$CLK_TCK" >> $OPENSEARCH_HOME/config/jvm.options - echo "-Djdk.attach.allowAttachSelf=true" >> $OPENSEARCH_HOME/config/jvm.options - echo "-Djava.security.policy=$OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_config/opensearch_security.policy" >> $OPENSEARCH_HOME/config/jvm.options -fi -echo "done plugins" - -##Check KNN lib existence in OpenSearch TAR distribution -echo "Checking kNN library" -FILE=`ls $KNN_LIB_DIR/libKNNIndex*.so` -if test -f "$FILE"; then - echo "FILE EXISTS $FILE" -else - echo "TEST FAILED OR FILE NOT EXIST $FILE" -fi - -##Set KNN Dylib Path for macOS and *nix systems -if echo "$OSTYPE" | grep -qi "darwin"; then - if echo "$JAVA_LIBRARY_PATH" | grep -q "$KNN_LIB_DIR"; then - echo "KNN lib path has been set" - else - export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:$KNN_LIB_DIR - echo "KNN lib path not found, set new path" - echo $JAVA_LIBRARY_PATH - fi -else - if echo "$LD_LIBRARY_PATH" | grep -q "$KNN_LIB_DIR"; then - echo "KNN lib path has been set" - else - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$KNN_LIB_DIR - echo "KNN lib path not found, set new path" - echo $LD_LIBRARY_PATH - fi -fi - -##Start OpenSearch -bash $OPENSEARCH_HOME/bin/opensearch "$@" diff --git a/release-tools/scripts/setup_runners.sh b/release-tools/scripts/setup_runners.sh deleted file mode 100755 index d042a326ea..0000000000 --- a/release-tools/scripts/setup_runners.sh +++ /dev/null @@ -1,264 +0,0 @@ -#!/bin/bash - -###### Information ############################################################################ -# Name: setup_runners.sh -# -# About: 1. Run instances on EC2 based on parameters defined and wait for completion -# 2. SSH to these instances and configure / bootstrap on $GIT_URL_REPO as runners -# 3. Unbootstrap the runners and terminate the instances for cleanups -# -# Usage: ./setup_runners.sh $ACTION $EC2_INSTANCE_NAMES $GITHUB_TOKEN -# $ACTION: run | terminate (required) -# $EC2_INSTANCE_NAMES: (required, sep ",") -# $GITHUB_TOKEN: GitHub PAT with repo scope and Admin Access to $GIT_URL_REPO -# -# Requirements: The env that runs this script must have its AWS resources with these configurations -# -# 1. Have an AWS user account with access to EC2 resource, remember the User ID -# -# 2. Create EC2 keypairs with name "odfe-release-runner" -# -# 3. Create EC2 Security Group with name "odfe-release-runner" -# with inbound rules of 22/9200/9600/5601 from IP ranges that need access to the runner -# -# 4. Create IAM resources: -# -# * IAM role with name "odfe-release-runner", and these policies attached to it: -# i. AmazonEC2RoleforSSM -# ii. AmazonSSMManagedInstanceCore -# -# * IAM user "opendistro-ec2-user", generate a pair of security credentials, -# and these policies attached to it: -# i. AmazonEC2FullAccess -# ii. Custom policy using this json, I name it again to "odfe-release-runner" -# { -# "Version": "2012-10-17", -# "Statement": [ -# { -# "Sid": "VisualEditor0", -# "Effect": "Allow", -# "Action": [ -# "ssm:SendCommand", -# "iam:PassRole" -# ], -# "Resource": [ -# "arn:aws:ssm:*:*:document/*", -# "arn:aws:ec2:*:*:instance/*", -# "arn:aws:iam:::role/" -# ] -# }, -# { -# "Sid": "VisualEditor1", -# "Effect": "Allow", -# "Action": "ssm:DescribeInstanceInformation", -# "Resource": "*" -# } -# ] -# } -# -# 5. awscli must "aws login" with the security credencial created for IAM user -# in the step 4 above -# -# 6. If you change the above resources name from "odfe-release-runner" to "xyz", -# please update "Variables / Parameters / Settings" section of this script -# -# 7. Runner AMI requires installation of packages of these (java version can be different as gradle might request a higher version): -# Debian: -# sudo apt install -y curl wget unzip tar jq python python3 git awscli openjdk-14-jdk -# sudo apt install -y libgtk2.0-0 libgtk-3-0 libgbm-dev libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 libxtst6 xauth xvfb -# -# RedHat: -# sudo yum install -y curl wget unzip tar jq python python3 git awscli java-latest-openjdk -# sudo yum install -y xorg-x11-server-Xvfb gtk2-devel gtk3-devel libnotify-devel GConf2 nss libXScrnSaver alsa-lib -# -# Also you need to install java devel if you want to compile library (e.g. knnlib) -# -# 8. AMI must be at least 16GB during the creation. -# -# 9. You can use `export GIT_UTL_REPO="opendistro-for-elasticsearch/opendistro-build"` or similar to set the Git Repo of the runner -# -# 10. JDK & SSM Agent -# You should find a way to install JDK14 or later on the server -# Dibian with: sudo add-apt-repository ppa:openjdk-r/ppa -# RedHat with: https://fedoraproject.org/wiki/EPEL -# -# Also, you need to install ssm agent -# on non-al2 machine due to ssm RunCommand requires that -# https://docs.aws.amazon.com/systems-manager/latest/userguide/sysman-manual-agent-install.html -# -# us-west-2 -# RPM x64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/linux_amd64/amazon-ssm-agent.rpm -# RPM arm64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/linux_arm64/amazon-ssm-agent.rpm -# DEB x64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/debian_amd64/amazon-ssm-agent.deb -# DEB arm64: https://s3.us-west-2.amazonaws.com/amazon-ssm-us-west-2/latest/debian_arm64/amazon-ssm-agent.deb -# yum or dpkg then systemctl enable/start amazon-ssm-agent -# -# 11. You also need to set the user of the GitHub Token to have ADMIN access of the GitHub Repo -# So that runner can be successfully bootstrapped to action tab in settings. -# -############################################################################################### - -set -e - -##################################### -# Variables / Parameters / Settings # -##################################### - -# This script allows users to manually assign parameters -if [ "$#" -lt 3 ] || [ "$1" = "--help" ] || [ "$1" = "-h" ] -then - echo "Please assign at least 3 parameters when running this script" - echo "Example: $0 \$ACTION \$EC2_INSTANCE_NAMES(,) \$GITHUB_TOKEN, \$EC2_AMI_ID" - echo "Example (run must have 4 parameters): $0 \"run\" \"opensearch-rpm-im,opensearch-rpm-sql\" \"\" \"ami-*\"" - echo "Example (terminate must have 3 parameters): $0 \"terminate\" \"opensearch-rpm-im,opensearch-rpm-sql\" \"\"" - echo "You can use \`export GIT_UTL_REPO=\"opendistro-for-elasticsearch/opendistro-build\"\` or similar to set the Git Repo of the runner" - exit 1 -fi - -SETUP_ACTION=$1 -SETUP_RUNNER=`echo $2 | sed 's/,/ /g'` -SETUP_GIT_TOKEN=$3 - -# AMI on us-west-2 -# Distro Arch Recommand Username AMI-ID Java Comments -# RPM-al2 x64 YES ec2-user ami-0bd968fea932935f4 none no jdk + reports kibana dependencies -# RPM-al2 arm64 YES ec2-user ami-0ef0c96643bbd01f2 jdk14 preinstall with tar.gz + reports kibana dependencies -# DEB-ubu1804 arm64 YES ubuntu ami-03f8a33a16290a84c jdk14 preinstall + docker + docker compose + reports kibana dependencies -# RPM-centos8 x64 NO centos ami-011f59f50bac33376 jdk15 preinstall -# RPM-centos8 arm64 NO centos ami-0ed17173ab64255b1 jdk15 preinstall -EC2_AMI_ID=$4 - -if [ "$SETUP_ACTION" = "run" ] -then - if [ -z "$EC2_AMI_ID" ] - then - echo " \$EC2_AMI_ID is empty, please add a 4th parameter for the run " - exit 1 - else - # This does not support MacOS now due to cumbersome descriptions - # MacOS sample: ami-00b3e436dc75183e0 - # "PlatformDetails": "Linux/UNIX" - # "Architecture": "x86_64_mac" - EC2_AMI_PLATFORM=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].PlatformDetails' --output text | awk -F '/' '{print $1}' | tr '[:upper:]' '[:lower:]'` - EC2_AMI_ARCH=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].Architecture' --output text | sed 's/x86_64/x64/g'` - EC2_AMI_NAME=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].Name' --output text | tr '[:upper:]' '[:lower:]'` - EC2_AMI_USER="ec2-user"; if echo $EC2_AMI_NAME | grep "centos"; then EC2_AMI_USER="centos"; elif echo $EC2_AMI_NAME | grep "ubuntu"; then EC2_AMI_USER="ubuntu"; fi - EC2_INSTANCE_TYPE="m5.xlarge"; if [ "$EC2_AMI_ARCH" = "arm64" ]; then EC2_INSTANCE_TYPE="m6g.xlarge"; fi - RUNNER_URL=`curl -s https://api.github.com/repos/actions/runner/releases/latest -H "Authorization: token $SETUP_GIT_TOKEN" | jq -r '.assets[].browser_download_url' | grep "$EC2_AMI_PLATFORM" | grep "$EC2_AMI_ARCH" | tail -n 1` - echo Provision $EC2_AMI_PLATFORM $EC2_AMI_ARCH $EC2_AMI_NAME $EC2_AMI_USER $EC2_INSTANCE_TYPE $RUNNER_URL - fi -fi - - -EC2_INSTANCE_SIZE=20 #GiB -EC2_KEYPAIR="odfe-release-runner" -EC2_SECURITYGROUP="odfe-release-runner" -IAM_ROLE="odfe-release-runner" -GIT_URL_API="https://api.github.com/repos" -GIT_URL_BASE="https://github.com" -GIT_URL_REPO=${GIT_URL_REPO:-opensearch-project/opensearch-build} -RUNNER_DIR="actions-runner" - - -echo "###############################################" -echo "Start Running $0 $1 $2" -echo "###############################################" - -############################################### -# Run / Start instances and bootstrap runners # -############################################### -if [ "$SETUP_ACTION" = "run" ] -then - echo "GIT_URL_REPO $GIT_URL_REPO" - - echo "" - echo "Run / Start instances and bootstrap runners [${SETUP_RUNNER}]" - echo "" - - # Get information - instance_root_device=`aws ec2 describe-images --image-id $EC2_AMI_ID --query 'Images[*].RootDeviceName' --output text` - - # Provision VMs - for instance_name1 in $SETUP_RUNNER - do - echo "[${instance_name1}]: Start provisioning vm" - aws ec2 run-instances --image-id $EC2_AMI_ID --count 1 --instance-type $EC2_INSTANCE_TYPE \ - --block-device-mapping DeviceName=$instance_root_device,Ebs={VolumeSize=$EC2_INSTANCE_SIZE} \ - --key-name $EC2_KEYPAIR --security-groups $EC2_SECURITYGROUP \ - --iam-instance-profile Name=$IAM_ROLE \ - --tag-specifications "ResourceType=instance,Tags=[{Key=Name,Value=$instance_name1}]" > /dev/null 2>&1; echo $? - sleep 1 - done - - echo "" - echo "Sleep for 120 seconds for EC2 instances to start running" - echo "" - - sleep 120 - - # Setup VMs to register as runners - for instance_name2 in $SETUP_RUNNER - do - echo "[${instance_name2}]: Make change of the runner hostname" - aws ssm send-command --targets Key=tag:Name,Values=$instance_name2 --document-name "AWS-RunShellScript" \ - --parameters '{"commands": ["#!/bin/bash", "sudo hostnamectl set-hostname '${instance_name2}'"]}' \ - --output text > /dev/null 2>&1; echo $? - - echo "[${instance_name2}]: Get latest runner binary to server ${RUNNER_URL}" - aws ssm send-command --targets Key=tag:Name,Values=$instance_name2 --document-name "AWS-RunShellScript" \ - --parameters '{"commands": ["#!/bin/bash", "sudo su - '${EC2_AMI_USER}' -c \"mkdir -p '${RUNNER_DIR}' && cd '${RUNNER_DIR}' && wget -q '${RUNNER_URL}' && tar -xzf *.tar.gz && rm *.tar.gz \""]}' \ - --output text > /dev/null 2>&1; echo $? - - echo "[${instance_name2}]: Get runner token and bootstrap on Git" - instance_runner_token=`curl --silent -H "Authorization: token ${SETUP_GIT_TOKEN}" --request POST "${GIT_URL_API}/${GIT_URL_REPO}/actions/runners/registration-token" | jq -r .token` - # Wait 10 seconds for untar of runner binary to complete - aws ssm send-command --targets Key=tag:Name,Values=$instance_name2 --document-name "AWS-RunShellScript" \ - --parameters '{"commands": ["#!/bin/bash", "sudo su - '${EC2_AMI_USER}' -c \"sleep 30 && cd '${RUNNER_DIR}' && ./config.sh --unattended --url '${GIT_URL_BASE}/${GIT_URL_REPO}' --labels '${instance_name2}' --token '${instance_runner_token}' && nohup ./run.sh &\""]}' \ - --output text > /dev/null 2>&1; echo $? - sleep 5 - done - - echo "" - echo "Wait for 90 seconds for runners to bootstrap on Git" - echo "" - - sleep 90 - - echo "" - echo "All runners are online on Git" - echo "" -fi - - -################################################### -# Terminate / Delete instances and remove runners # -################################################### -if [ "$SETUP_ACTION" = "terminate" ] -then - echo "GIT_URL_REPO $GIT_URL_REPO" - - echo "" - echo "Terminate / Delete instances and remove runners [${SETUP_RUNNER}]" - echo "" - - for instance_name3 in $SETUP_RUNNER - do - instance_runner_id_git=`curl --silent -H "Authorization: token ${SETUP_GIT_TOKEN}" --request GET "${GIT_URL_API}/${GIT_URL_REPO}/actions/runners" | jq ".runners[] | select(.name == \"${instance_name3}\") | .id"` - echo "[${instance_name3}]: Unbootstrap runner from Git" - curl --silent -H "Authorization: token ${SETUP_GIT_TOKEN}" --request DELETE "${GIT_URL_API}/${GIT_URL_REPO}/actions/runners/${instance_runner_id_git}"; echo $? - - instance_runner_id_ec2=`aws ec2 describe-instances --filters "Name=tag:Name,Values=$instance_name3" | jq -r '.Reservations[].Instances[] | select(.State.Code == 16) | .InstanceId'` # Only running instances - echo "[${instance_name3}]: Remove tags Name" - aws ec2 delete-tags --resources $instance_runner_id_ec2 --tags Key=Name > /dev/null 2>&1; echo $? - - echo "[${instance_name3}]: Terminate runner" - aws ec2 terminate-instances --instance-ids $instance_runner_id_ec2 > /dev/null 2>&1; echo $? - - sleep 1 - done - - echo "All runners are offline on Git" -fi - - - diff --git a/standalone-tools/vulnerability-scan/wss-scan.config b/standalone-tools/vulnerability-scan/wss-scan.config deleted file mode 100644 index 30d7e64d2f..0000000000 --- a/standalone-tools/vulnerability-scan/wss-scan.config +++ /dev/null @@ -1,3 +0,0 @@ -baseDirPath=$(pwd) -gitBasePath=https://github.com/opensearch-project/ -gitRepos=alerting,alerting-dashboards-plugin,anomaly-detection,anomaly-detection-dashboards-plugin,asynchronous-search,common-utils,dashboards-notebooks,dashboards-reports,dashboards-visualizations,data-prepper,index-management,index-management-dashboards-plugin,job-scheduler,k-NN,opensearch-cli,performance-analyzer,performance-analyzer-rca,perftop,security,security-dashboards-plugin,sql,trace-analytics diff --git a/standalone-tools/vulnerability-scan/wss-scan.sh b/standalone-tools/vulnerability-scan/wss-scan.sh deleted file mode 100755 index 97c9c318b6..0000000000 --- a/standalone-tools/vulnerability-scan/wss-scan.sh +++ /dev/null @@ -1,105 +0,0 @@ -#!/bin/bash -###### Information ############################################################################ -# Name: wss-scan.sh -# Language: Shell -# -# About: This script is to scan the OpenSearch distros for vulnerabilities and licenses -# It will scan the repositories and send the WhiteSource link to the mail -# of the user. -# -# Prerequisites: Need to install Java 11 -# Export JAVA_HOME env variable to the JDK path -# Add JAVA_HOME to PATH variable -# Need to set the recepient mail in wss-scan.config for local run -# WhiteSource API key is needed for local run, The API Key can be retrieved from the -# WhiteSource Admin Console of your account.Use the below command to export the API key -# export wss_apikey=$(APIKEY) -# -# Usage: ./wss-scan.sh -# -############################################################################################### - -set -e - -java -version -if [ "$?" != 0 ] -then - echo "Java has not been setup" - exit 1 -fi - -if [ ! -f "wss-unified-agent.jar" ] -then - # Download the WhiteSource Agent - wget -q https://github.com/whitesource/unified-agent-distribution/releases/latest/download/wss-unified-agent.jar - # The version 20.9.2.1 has been tested and can be used if a specific version is required - #wget -q https://github.com/whitesource/unified-agent-distribution/releases/download/v20.9.2.1/wss-unified-agent.jar -fi - -# scan the config file for the user configurations -# wss-scan.config has to be present in the same working directory as the script -source wss-scan.config - -# change comma to whitespace -gitRepos=${gitRepos//,/$'\n'} - -basepath=$baseDirPath"/repos" - -echo "Cleaning up scan directories if already present" -rm -rf $basepath - -mkdir -p $basepath - - -# clone the desired Repos for scanning -for repo in $gitRepos -do - echo "Cloning repo "$gitBasePath$repo - git clone "$gitBasePath$repo".git $basepath"/"$repo -done - -echo -n > info.txt - - -# scan the Repos using the WhiteSource Unified Agent -for repo in $gitRepos -do - repo_path=$basepath"/"$repo - if [ -d "$repo_path" ] - then - echo "Scanning repo: "$gitBasePath$repo " Project: " $repo - java -jar wss-unified-agent.jar -c wss-unified-agent.config -d $repo_path -apiKey $wss_apikey -product OpenSearch -project $repo | grep "Project name" | sed 's/^.\{,41\}//' >> info.txt 2>&1 - else - echo "Scanning failed for repo: "$gitBasePath$repo " Project: " $repo - fi -done - - - -# mail function to send the scan details to the desired recepient -mail_format_func() -{ - -echo "" > output.md -while IFS= read -r line -do -# setting comma as the delimiter - - IFS=',' - read -ra val <<< "$line" - echo "" >> output.md - for ln in "${val[@]}" - do - echo "${ln//[[:space:]]/}" - echo "" >> output.md - done - echo "" >> output.md -done < info.txt -echo "
"${ln//[[:space:]]/}"
" >> output.md - -} - -mail_format_func - -# remove the WhiteSource unified Jar -rm "wss-unified-agent.jar" diff --git a/standalone-tools/vulnerability-scan/wss-unified-agent.config b/standalone-tools/vulnerability-scan/wss-unified-agent.config deleted file mode 100644 index 6d50d45e7e..0000000000 --- a/standalone-tools/vulnerability-scan/wss-unified-agent.config +++ /dev/null @@ -1,363 +0,0 @@ -############################################################### -# WhiteSource Unified-Agent configuration file -############################################################### -# GENERAL SCAN MODE: Files and Package Managers -############################################################### -# Organization vitals -###################### - -#apiKey='${wss_apikey}' -apiKey= -#userKey is required if WhiteSource administrator has enabled "Enforce user level access" option -#userKey= -#requesterEmail=user@provider.com - -projectName= -projectVersion= -projectToken= -#projectTag= key:value - -productName= -productVersion= -productToken= - -#projectPerFolder=true -#projectPerFolderIncludes= -#projectPerFolderExcludes= - -#wss.connectionTimeoutMinutes=60 - -# Change the below URL to your WhiteSource server. -# Use the 'WhiteSource Server URL' which can be retrieved -# from your 'Profile' page on the 'Server URLs' panel. -# Then, add the '/agent' path to it. -wss.url=https://saas.whitesourcesoftware.com/agent -#wss.url=https://app.whitesourcesoftware.com/agent -#wss.url=https://app-eu.whitesourcesoftware.com/agent - -############ -# Policies # -############ -checkPolicies=false -forceCheckAllDependencies=false -forceUpdate=false -forceUpdate.failBuildOnPolicyViolation=false -#updateInventory=false - -########### -# General # -########### -#offline=false -#updateType=APPEND -#ignoreSourceFiles=true -#scanComment= -#failErrorLevel=ALL -#requireKnownSha1=false - -#generateProjectDetailsJson=true -#generateScanReport=true -#scanReportTimeoutMinutes=10 -#scanReportFilenameFormat= - -#analyzeFrameworks=true -#analyzeFrameworksReference= - -#updateEmptyProject=false - -#log.files.level= -#log.files.maxFileSize= -#log.files.maxFilesCount= -#log.files.path= - -######################################## -# Package Manager Dependency resolvers # -######################################## -resolveAllDependencies=false -#excludeDependenciesFromNodes=.*commons-io.*,.*maven-model - -#npm.resolveDependencies=false -#npm.ignoreSourceFiles=false -#npm.includeDevDependencies=true -#npm.runPreStep=true -#npm.ignoreNpmLsErrors=true -#npm.ignoreScripts=true -#npm.yarnProject=true -#npm.accessToken= -#npm.identifyByNameAndVersion=true -#npm.yarn.frozenLockfile=true -#npm.resolveMainPackageJsonOnly=true -#npm.removeDuplicateDependencies=false -#npm.resolveAdditionalDependencies=true -#npm.failOnNpmLsErrors = -#npm.projectNameFromDependencyFile = true -#npm.resolveGlobalPackages=true -#npm.resolveLockFile=true - -#bower.resolveDependencies=false -#bower.ignoreSourceFiles=true -#bower.runPreStep=true - -#nuget.resolvePackagesConfigFiles=false -#nuget.resolveCsProjFiles=false -#nuget.resolveDependencies=false -#nuget.restoreDependencies=true -#nuget.preferredEnvironment= -#nuget.packagesDirectory= -#nuget.ignoreSourceFiles=false -#nuget.runPreStep=true -#nuget.resolveNuspecFiles=false -#nuget.resolveAssetsFiles=true - -#python.resolveDependencies=false -#python.ignoreSourceFiles=false -#python.ignorePipInstallErrors=true -#python.installVirtualenv=true -#python.resolveHierarchyTree=false -#python.requirementsFileIncludes=requirements.txt -#python.resolveSetupPyFiles=true -#python.runPipenvPreStep=true -#python.pipenvDevDependencies=true -#python.IgnorePipenvInstallErrors=true -#python.resolveGlobalPackages=true -#python.localPackagePathsToInstall=/path/to/local/dependency.egg, /path/to/local/dependency.zip -#python.resolvePipEditablePackages -#python.path=/path/to/python -#python.pipPath=/path/to/pip -#python.runPoetryPreStep=true -#python.includePoetryDevDependencies=true - -#maven.ignoredScopes=test provided -#maven.resolveDependencies=false -#maven.ignoreSourceFiles=true -#maven.aggregateModules=true -#maven.ignorePomModules=false -#maven.runPreStep=true -#maven.ignoreMvnTreeErrors=true -#maven.environmentPath= -#maven.m2RepositoryPath= -#maven.downloadMissingDependencies=false -#maven.additionalArguments= -#maven.projectNameFromDependencyFile=true - -resolveAllDependencies=false -archiveExtractionDepth=7 -followSymbolicLinks=true -gradle.resolveDependencies=true -gradle.aggregateModules=true -maven.resolveDependencies=true -maven.runPreStep=true -maven.aggregateModules=true -maven.ignoredScopes=None -npm.resolveDependencies=true -npm.runPreStep=true -npm.yarnProject=true -go.collectDependenciesAtRuntime=true -go.dependencyManager=modules -go.resolveDependencies=true - -#gradle.ignoredScopes= -#gradle.resolveDependencies=true -#gradle.runAssembleCommand=true -#gradle.runPreStep=true -#gradle.ignoreSourceFiles=true -#gradle.aggregateModules=true -#gradle.preferredEnvironment=wrapper -#gradle.localRepositoryPath= -#gradle.wrapperPath= -#gradle.downloadMissingDependencies=false -#gradle.additionalArguments= -#gradle.includedScopes= -#gradle.excludeModules= -#gradle.includeModules= -#gradle.includedConfigurations= -#gradle.ignoredConfigurations= - -#paket.resolveDependencies=false -#paket.ignoredGroups= -#paket.ignoreSourceFiles=false -#paket.runPreStep=true -#paket.exePath= - -#go.resolveDependencies=false -#go.collectDependenciesAtRuntime=true -#go.dependencyManager= -#go.ignoreSourceFiles=true -#go.glide.ignoreTestPackages=false -#go.gogradle.enableTaskAlias=true - -#ruby.resolveDependencies=false -#ruby.ignoreSourceFiles=false -#ruby.installMissingGems=true -#ruby.runBundleInstall=true -#ruby.overwriteGemFile=true - -#sbt.resolveDependencies=false -#sbt.ignoreSourceFiles=true -#sbt.aggregateModules=true -#sbt.runPreStep=true -#sbt.includedScopes= - -#php.resolveDependencies=false -#php.runPreStep=true -#php.includeDevDependencies=true - -#html.resolveDependencies=false - -#cocoapods.resolveDependencies=false -#cocoapods.runPreStep=true -#cocoapods.ignoreSourceFiles=false - -#hex.resolveDependencies=false -#hex.runPreStep=true -#hex.ignoreSourceFiles=false -#hex.aggregateModules=true - -#ant.resolveDependencies=false -#ant.pathIdIncludes=.* -#ant.external.parameters= - -#r.resolveDependencies=false -#r.runPreStep=true -#r.ignoreSourceFiles=false -#r.cranMirrorUrl= -#r.packageManager=None - -#cargo.resolveDependencies=false -#cargo.runPreStep=true -#cargo.ignoreSourceFiles=false - -#haskell.resolveDependencies=false -#haskell.runPreStep=true -#haskell.ignoreSourceFiles=false -#haskell.ignorePreStepErrors=true - -#ocaml.resolveDependencies=false -#ocaml.runPrepStep=true -#ocaml.ignoreSourceFiles=false -#ocaml.switchName= -#ocaml.ignoredScopes=none -#ocaml.aggregateModules=true - -#bazel.resolveDependencies=false -#bazel.runPrepStep=true - -########################################################################################### -# Includes/Excludes Glob patterns - Please use only one exclude line and one include line # -########################################################################################### -includes=**/*.c **/*.cc **/*.cp **/*.cpp **/*.cxx **/*.c++ **/*.h **/*.hpp **/*.hxx **/*.jar **/*.java - -#includes=**/*.m **/*.mm **/*.js **/*.php -#includes=**/*.jar -#includes=**/*.gem **/*.rb -#includes=**/*.dll **/*.cs **/*.nupkg -#includes=**/*.tgz **/*.deb **/*.gzip **/*.rpm **/*.tar.bz2 -#includes=**/*.zip **/*.tar.gz **/*.egg **/*.whl **/*.py - -#Exclude file extensions or specific directories by adding **/*. or **//** -excludes=**/*sources.jar **/*javadoc.jar - -case.sensitive.glob=false -followSymbolicLinks=true - -###################### -# Archive properties # -###################### -#archiveExtractionDepth=2 -#archiveIncludes=**/*.war **/*.ear -#archiveExcludes=**/*sources.jar - -############## -# SCAN MODES # -############## - -# Docker images -################ -#docker.scanImages=true -#docker.includes=.*.* -#docker.excludes= -#docker.pull.enable=true -#docker.pull.images=.*.* -#docker.pull.maxImages=10 -#docker.pull.tags=.*.* -#docker.pull.digest= -#docker.delete.force=true -#docker.login.sudo=false -#docker.projectNameFormat={repositoryNameAndTag|repositoryName|default} -#docker.scanTarFiles=true - -#docker.aws.enable=true -#docker.aws.registryIds= - -#docker.azure.enable=true -#docker.azure.userName= -#docker.azure.userPassword= -#docker.azure.registryNames= -#docker.azure.authenticationType=containerRegistry -#docker.azure.registryAuthenticationParameters=: : - -#docker.gcr.enable=true -#docker.gcr.account= -#docker.gcr.repositories= - -#docker.artifactory.enable=true -#docker.artifactory.url= -#docker.artifactory.pullUrl= -#docker.artifactory.userName= -#docker.artifactory.userPassword= -#docker.artifactory.repositoriesNames= -#docker.artifactory.dockerAccessMethod= - -#docker.hub.enabled=true -#docker.hub.userName= -#docker.hub.userPassword= -#docker.hub.organizationsNames= - -# Docker containers -#################### -#docker.scanContainers=true -#docker.containerIncludes=.*.* -#docker.containerExcludes= - -# Linux package manager settings -################################ -#scanPackageManager=true - -# Serverless settings -###################### -#serverless.provider= -#serverless.scanFunctions=true -#serverless.includes= -#serverless.excludes= -#serverless.region= -#serverless.maxFunctions=10 - -# Artifactory settings -######################## -#artifactory.enableScan=true -#artifactory.url= -#artifactory.accessToken= -#artifactory.repoKeys= -#artifactory.userName= -#artifactory.userPassword= - -################## -# Proxy settings # -################## -#proxy.host= -#proxy.port= -#proxy.user= -#proxy.pass= - -################ -# SCM settings # -################ -#scm.type= -#scm.user= -#scm.pass= -#scm.ppk= -#scm.url= -#scm.branch= -#scm.tag= -#scm.npmInstall= -#scm.npmInstallTimeoutMinutes= -#scm.repositoriesFile= From 3f3a138d782a206a65e8f300355768996ed509ef Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 3 May 2021 16:33:17 -0400 Subject: [PATCH 3/5] Tweak config folders Signed-off-by: Peter Zhu --- config/{ => opensearch-dashboards}/opensearch_dashboards.yml | 0 config/{ => opensearch}/opensearch.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename config/{ => opensearch-dashboards}/opensearch_dashboards.yml (100%) rename config/{ => opensearch}/opensearch.yml (100%) diff --git a/config/opensearch_dashboards.yml b/config/opensearch-dashboards/opensearch_dashboards.yml similarity index 100% rename from config/opensearch_dashboards.yml rename to config/opensearch-dashboards/opensearch_dashboards.yml diff --git a/config/opensearch.yml b/config/opensearch/opensearch.yml similarity index 100% rename from config/opensearch.yml rename to config/opensearch/opensearch.yml From 59e22e6fd48729109b669b2833abb91b374fc1af Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 3 May 2021 16:34:34 -0400 Subject: [PATCH 4/5] Add tar release Signed-off-by: Peter Zhu --- release/tar/linux/opensearch-tar-install.sh | 73 +++++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100755 release/tar/linux/opensearch-tar-install.sh diff --git a/release/tar/linux/opensearch-tar-install.sh b/release/tar/linux/opensearch-tar-install.sh new file mode 100755 index 0000000000..d51781facf --- /dev/null +++ b/release/tar/linux/opensearch-tar-install.sh @@ -0,0 +1,73 @@ +#!/bin/bash + +# Copyright 2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# You may not use this file except in compliance with the License. +# A copy of the License is located at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# or in the "license" file accompanying this file. This file is distributed +# on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either +# express or implied. See the License for the specific language governing +# permissions and limitations under the License. + +OPENSEARCH_HOME=`dirname $(realpath $0)`; cd $OPENSEARCH_HOME +KNN_LIB_DIR=$OPENSEARCH_HOME/plugins/opensearch-knn/knnlib +##Security Plugin +bash $OPENSEARCH_HOME/plugins/opensearch-security/tools/install_demo_configuration.sh -y -i -s + +##Perf Plugin +chmod 755 $OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_bin/performance-analyzer-agent +chmod -R 755 /dev/shm +chmod 755 $OPENSEARCH_HOME/bin/performance-analyzer-agent-cli +echo "done security" +PA_AGENT_JAVA_OPTS="-Dlog4j.configurationFile=$OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_config/log4j2.xml \ + -Xms64M -Xmx64M -XX:+UseSerialGC -XX:CICompilerCount=1 -XX:-TieredCompilation -XX:InitialCodeCacheSize=4096 \ + -XX:InitialBootClassLoaderMetaspaceSize=30720 -XX:MaxRAM=400m" + +OPENSEARCH_MAIN_CLASS="com.amazon.opendistro.opensearch.performanceanalyzer.PerformanceAnalyzerApp" \ +OPENSEARCH_ADDITIONAL_CLASSPATH_DIRECTORIES=plugins/opensearch-performance-analyzer \ +OPENSEARCH_JAVA_OPTS=$PA_AGENT_JAVA_OPTS + +if ! grep -q '## OpenSearch Performance Analyzer' $OPENSEARCH_HOME/config/jvm.options; then + CLK_TCK=`/usr/bin/getconf CLK_TCK` + echo >> $OPENSEARCH_HOME/config/jvm.options + echo '## OpenSearch Performance Analyzer' >> $OPENSEARCH_HOME/config/jvm.options + echo "-Dclk.tck=$CLK_TCK" >> $OPENSEARCH_HOME/config/jvm.options + echo "-Djdk.attach.allowAttachSelf=true" >> $OPENSEARCH_HOME/config/jvm.options + echo "-Djava.security.policy=$OPENSEARCH_HOME/plugins/opensearch-performance-analyzer/pa_config/opensearch_security.policy" >> $OPENSEARCH_HOME/config/jvm.options +fi +echo "done plugins" + +##Check KNN lib existence in OpenSearch TAR distribution +echo "Checking kNN library" +FILE=`ls $KNN_LIB_DIR/libKNNIndex*.so` +if test -f "$FILE"; then + echo "FILE EXISTS $FILE" +else + echo "TEST FAILED OR FILE NOT EXIST $FILE" +fi + +##Set KNN Dylib Path for macOS and *nix systems +if echo "$OSTYPE" | grep -qi "darwin"; then + if echo "$JAVA_LIBRARY_PATH" | grep -q "$KNN_LIB_DIR"; then + echo "KNN lib path has been set" + else + export JAVA_LIBRARY_PATH=$JAVA_LIBRARY_PATH:$KNN_LIB_DIR + echo "KNN lib path not found, set new path" + echo $JAVA_LIBRARY_PATH + fi +else + if echo "$LD_LIBRARY_PATH" | grep -q "$KNN_LIB_DIR"; then + echo "KNN lib path has been set" + else + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$KNN_LIB_DIR + echo "KNN lib path not found, set new path" + echo $LD_LIBRARY_PATH + fi +fi + +##Start OpenSearch +bash $OPENSEARCH_HOME/bin/opensearch "$@" From 854bf8eeec9e90f88faf6517101072e6b419e013 Mon Sep 17 00:00:00 2001 From: Peter Zhu Date: Mon, 3 May 2021 16:59:06 -0400 Subject: [PATCH 5/5] Reset the config folder structure of the original Signed-off-by: Peter Zhu --- config/opensearch/opensearch.yml | 7 ------- .../{opensearch-dashboards => }/opensearch_dashboards.yml | 0 2 files changed, 7 deletions(-) delete mode 100644 config/opensearch/opensearch.yml rename config/{opensearch-dashboards => }/opensearch_dashboards.yml (100%) diff --git a/config/opensearch/opensearch.yml b/config/opensearch/opensearch.yml deleted file mode 100644 index 748b2235e5..0000000000 --- a/config/opensearch/opensearch.yml +++ /dev/null @@ -1,7 +0,0 @@ -cluster.name: docker-cluster - -# Bind to all interfaces because we don't know what IP address Docker will assign to us. -network.host: 0.0.0.0 - -# Setting network.host to a non-loopback address enables the annoying bootstrap checks. "Single-node" mode disables them again. -discovery.type: single-node diff --git a/config/opensearch-dashboards/opensearch_dashboards.yml b/config/opensearch_dashboards.yml similarity index 100% rename from config/opensearch-dashboards/opensearch_dashboards.yml rename to config/opensearch_dashboards.yml