diff --git a/DemosOperator/Hack/README.md b/DemosOperator/Hack/README.md index 05ac14fb7..f89d87f98 100644 --- a/DemosOperator/Hack/README.md +++ b/DemosOperator/Hack/README.md @@ -98,7 +98,6 @@ spec: # demo will automatically enable required addons. addons: postgres: false - elasticSearch: false # Installs the pipelines operator cluster scoped ocpPipelines: false EOF diff --git a/EventEnabledInsurance/kafkaconnect/connector-elastic.yaml b/EventEnabledInsurance/kafkaconnect/connector-elastic.yaml deleted file mode 100644 index fe6088476..000000000 --- a/EventEnabledInsurance/kafkaconnect/connector-elastic.yaml +++ /dev/null @@ -1,29 +0,0 @@ -apiVersion: eventstreams.ibm.com/v1alpha1 -kind: KafkaConnector -metadata: - name: eei-elastic - labels: - eventstreams.ibm.com/cluster: eei-cluster -spec: - # This uses the Elasticsearch plugin from the KafkaConnectS2I - class: com.ibm.eventstreams.connect.elasticsink.ElasticSinkConnector - tasksMax: 1 - config: - # Monitors the topic that is being populated by the postgres connector. - topics: sor.public.quotes - # The following credentials refer to the mounted secret and use the FileConfigProvider - # from the KafkaConnectS2I to extract properties from the properties file. - es.connection: "${file:/opt/kafka/external-configuration/elastic-connector-config/connector.properties:dbConnection}" - es.user.name: "${file:/opt/kafka/external-configuration/elastic-connector-config/connector.properties:dbUser}" - es.password: "${file:/opt/kafka/external-configuration/elastic-connector-config/connector.properties:dbPassword}" - # Use the default document/index builders - es.document.builder: com.ibm.eventstreams.connect.elasticsink.builders.JsonDocumentBuilder - es.index.builder: com.ibm.eventstreams.connect.elasticsink.builders.DefaultIndexBuilder - # Use the KeyIdentifierBuilder to do CDC, so the Elasticsearch index only includes - # the latest copy of all rows from the original data. - es.identifier.builder: com.ibm.eventstreams.connect.elasticsink.builders.KeyIdentifierBuilder - # Setup the truststore to trust the Elasticsearch self signed certificate. The Elasticsearch - # operator creates this certificate and the prereqs create a jks truststore from it and - # add it to a secret that gets mounted into the connector pod. - es.tls.truststore.location: /opt/kafka/external-configuration/elastic-connector-config/elastic-ts.jks - es.tls.truststore.password: "${file:/opt/kafka/external-configuration/elastic-connector-config/connector.properties:truststorePassword}" diff --git a/EventEnabledInsurance/kafkaconnect/kafka-connect-s2i.yaml b/EventEnabledInsurance/kafkaconnect/kafka-connect-s2i.yaml index f38dee25b..0033c584a 100644 --- a/EventEnabledInsurance/kafkaconnect/kafka-connect-s2i.yaml +++ b/EventEnabledInsurance/kafkaconnect/kafka-connect-s2i.yaml @@ -43,15 +43,12 @@ spec: config.providers.file.class: org.apache.kafka.common.config.provider.FileConfigProvider # This mounts secrets into the connector at /opt/kafka/external-configuration. These # secrets have been pre-created by the prereqs.sh script and configure access to the - # demo installs of Postgres and Elasticsearch. + # demo install of Postgres. externalConfiguration: volumes: - name: postgres-connector-config secret: secretName: eei-postgres-replication-credential - - name: elastic-connector-config - secret: - secretName: eei-elastic-credential # There is no need to add tls or authentication properties, `es-demos` has no security # setup. # tls: diff --git a/EventEnabledInsurance/prereqs.sh b/EventEnabledInsurance/prereqs.sh index 32da5cc88..73b65b3c9 100755 --- a/EventEnabledInsurance/prereqs.sh +++ b/EventEnabledInsurance/prereqs.sh @@ -15,7 +15,6 @@ # -n : (string), Defaults to 'cp4i' # -r : (string), Defaults to 'https://github.com/IBM/cp4i-deployment-samples.git' # -b : (string), Defaults to 'main' -# -e : (string), Namespace for elastic search , Defaults to 'elasticsearch' # -p : (string), Namespace where postgres is setup, Defaults to the value of # -o : (optional), Parameter to decide if initial setup is to be done or not, Defaults to false # -f : (string), Default to 'ibmc-file-gold-gid' @@ -25,14 +24,14 @@ # ./prereqs.sh # # With overridden values -# ./prereqs.sh -n -r -b -e -p -f -g -o +# ./prereqs.sh -n -r -b -p -f -g -o function divider() { echo -e "\n-------------------------------------------------------------------------------------------------------------------\n" } function usage() { - echo "Usage: $0 -n -r -b -e -p -f -g [-o]" + echo "Usage: $0 -n -r -b -p -f -g [-o]" divider exit 1 } @@ -47,7 +46,6 @@ REPO="https://github.com/IBM/cp4i-deployment-samples.git" BRANCH="main" INFO="\xE2\x84\xB9" MISSING_PARAMS="false" -ELASTIC_NAMESPACE= OMIT_INITIAL_SETUP=false DEFAULT_FILE_STORAGE="ibmc-file-gold-gid" DEFAULT_BLOCK_STORAGE="ibmc-block-gold" @@ -64,7 +62,7 @@ while getopts "n:r:b:e:p:of:g:" opt; do BRANCH="$OPTARG" ;; e) - ELASTIC_NAMESPACE="$OPTARG" + echo "-e deprecated" ;; p) POSTGRES_NAMESPACE="$OPTARG" @@ -85,7 +83,6 @@ while getopts "n:r:b:e:p:of:g:" opt; do done POSTGRES_NAMESPACE=${POSTGRES_NAMESPACE:-$NAMESPACE} -ELASTIC_NAMESPACE=${ELASTIC_NAMESPACE:-$NAMESPACE} if [[ -z "${NAMESPACE// /}" ]]; then echo -e "$CROSS [ERROR] Namespace for event enabled insurance demo is empty. Please provide a value for '-n' parameter." @@ -102,11 +99,6 @@ if [[ -z "${BRANCH// /}" ]]; then MISSING_PARAMS="true" fi -if [[ -z "${ELASTIC_NAMESPACE// /}" ]]; then - echo -e "$CROSS [ERROR] Namespace for elastic search for event enabled insurance demo is empty. Please provide a value for '-e' parameter." - MISSING_PARAMS="true" -fi - if [[ -z "${POSTGRES_NAMESPACE// /}" ]]; then echo -e "$CROSS [ERROR] Namespace for postgres for event enabled insurance demo is empty. Please provide a value for '-p' parameter." MISSING_PARAMS="true" @@ -131,7 +123,6 @@ CURRENT_DIR=$(dirname $0) echo -e "$INFO [INFO] Current directory for the event enabled insurance demo: '$CURRENT_DIR'" echo -e "$INFO [INFO] Namespace for running event enabled insurance demo prereqs: '$NAMESPACE'" echo -e "$INFO [INFO] Namespace for postgres for the event enabled insurance demo: '$POSTGRES_NAMESPACE'" -echo -e "$INFO [INFO] Namespace for elastic search for the event enabled insurance demo: '$ELASTIC_NAMESPACE'" echo -e "$INFO [INFO] Suffix for the postgres for the event enabled insurance demo: '$SUFFIX'" echo -e "$INFO [INFO] Samples repository for the event enabled insurance demo: '$REPO'" echo -e "$INFO [INFO] Samples repo branch for the event enabled insurance demo: '$BRANCH'" @@ -364,16 +355,6 @@ $TKN clustertask ls | grep git-clone divider -echo -e "$INFO [INFO] Installing Elastic search operator and Elastic search instance..." -if ! $CURRENT_DIR/setup-elastic-search.sh -n $NAMESPACE -e $ELASTIC_NAMESPACE; then - echo -e "\n$CROSS [ERROR] Failed to install elastic search in the '$ELASTIC_NAMESPACE' namespace and configure it in the '$NAMESPACE' namespace" - exit 1 -else - echo -e "\n$TICK [SUCCESS] Successfully installed elastic search in the '$ELASTIC_NAMESPACE' namespace and configured it in the '$NAMESPACE' namespace" -fi #setup-elastic-search.sh - -divider - echo -e "$INFO [INFO] Building and deploying the EEI apps ..." if ! $CURRENT_DIR/build/build.sh -n $NAMESPACE -r $REPO -b $BRANCH -t $TKN -f "$DEFAULT_FILE_STORAGE" -g "$DEFAULT_BLOCK_STORAGE"; then echo -e "\n$CROSS [ERROR] Failed to build/deploy the EEI apps in the '$NAMESPACE' namespace" diff --git a/EventEnabledInsurance/readme.md b/EventEnabledInsurance/readme.md index 88c82427b..51774f895 100644 --- a/EventEnabledInsurance/readme.md +++ b/EventEnabledInsurance/readme.md @@ -17,8 +17,6 @@ The script carries out the following: - Creates a PUBLICATION named `DB_EEI_QUOTES` for the `QUOTES` table. (The Debezium connector can do this, but would then require super user privileges) - Creates a replication user that has the replication role and access to the `QUOTES` table - Creates a secret with the replication username/password that can be used by the `KafkaConnector` -- Installs Elasticsearch in the default `cp4i` project. The Elasticsearch CR is also setup to add a `subjectAltNames` so the self signed certificate can be used to access the service cross namespace. -- Creates a secret to allow the Elasticsearch connector to connect to Elasticsearch. This secret includes credentials and also a truststore in jks format. The truststore includes the self-signed certificate created by Elasticsearch. # Set up a Kafka Connect environment Download the [example kafka-connect-s2i.yaml](kafkaconnect/kafka-connect-s2i.yaml). This is based on the one in @@ -73,15 +71,12 @@ spec: config.providers.file.class: org.apache.kafka.common.config.provider.FileConfigProvider # This mounts secrets into the connector at /opt/kafka/external-configuration. These # secrets have been pre-created by the prereqs.sh script and configure access to the - # demo installs of Postgres and Elasticsearch. + # demo install of Postgres. externalConfiguration: volumes: - name: postgres-connector-config secret: secretName: eei-postgres-replication-credential - - name: elastic-connector-config - secret: - secretName: eei-elastic-credential # There is no need to add tls or authentication properties, `es-demos` has no security # setup. # tls: @@ -136,15 +131,13 @@ Status: ``` # Add connectors to your Kafka Connect environment -Add connectors for Postgres Debezium and Elasticsearch. +Add connector for Postgres Debezium. - Navigate to the toolbox for the `es-demo` Event Streams runtime - Click `Add connectors to your Kafka Connect environment` - Click `View Catalog`. - Find and click the following connectors and the click `Get connector` to download: - PostgreSQL (Debezium) - - Elasticsearch - Extract the PostgreSQL (Debezium) tgz into a dir named `my-plugins` -- Copy the Elasticsearch jar into the same `my-plugins` dir You should end up with a dir structure as follows: ![dir structure](./media/my-plugins-dir.png) @@ -165,9 +158,6 @@ section: $ oc describe KafkaConnectS2I eei-cluster ... Connector Plugins: - Class: com.ibm.eventstreams.connect.elasticsink.ElasticSinkConnector - Type: sink - Version: 1.0.1 Class: io.debezium.connector.postgresql.PostgresConnector Type: source Version: 1.2.0.Final @@ -301,143 +291,6 @@ Stop the Projection Claims application using: oc scale deployment/projection-claims-eei --replicas=0 ``` -# Start Kafka Connect with the Elasticsearch connector - -Download the [example connector-elastic.yaml](kafkaconnect/connector-elastic.yaml). This is based on the one in -the Event Streams toolbox, which can be accessed by: -- Navigate to the toolbox for the `es-demo` Event Streams runtime -- Click `Start Kafka Connect with your connectors` -- Jump to the `Start a connector` section. -- View the example connector.yaml - -The example includes comments describing each change, see the following: -```yaml -apiVersion: eventstreams.ibm.com/v1alpha1 -kind: KafkaConnector -metadata: - name: eei-elastic - labels: - eventstreams.ibm.com/cluster: eei-cluster -spec: - # This uses the Elasticsearch plugin from the KafkaConnectS2I - class: com.ibm.eventstreams.connect.elasticsink.ElasticSinkConnector - tasksMax: 1 - config: - # Monitors the topic that is being populated by the postgres connector. - topics: sor.public.quotes - # The following credentials refer to the mounted secret and use the FileConfigProvider - # from the KafkaConnectS2I to extract properties from the properties file. - es.connection: "${file:/opt/kafka/external-configuration/elastic-connector-config/connector.properties:dbConnection}" - es.user.name: "${file:/opt/kafka/external-configuration/elastic-connector-config/connector.properties:dbUser}" - es.password: "${file:/opt/kafka/external-configuration/elastic-connector-config/connector.properties:dbPassword}" - # Use the default document/index builders - es.document.builder: com.ibm.eventstreams.connect.elasticsink.builders.JsonDocumentBuilder - es.index.builder: com.ibm.eventstreams.connect.elasticsink.builders.DefaultIndexBuilder - # Use the KeyIdentifierBuilder to do CDC, so the Elasticsearch index only includes - # the latest copy of all rows from the original data. - es.identifier.builder: com.ibm.eventstreams.connect.elasticsink.builders.KeyIdentifierBuilder - # Setup the truststore to trust the Elasticsearch self signed certificate. The Elasticsearch - # operator creates this certificate and the prereqs create a jks truststore from it and - # add it to a secret that gets mounted into the connector pod. - es.tls.truststore.location: /opt/kafka/external-configuration/elastic-connector-config/elastic-ts.jks - es.tls.truststore.password: "${file:/opt/kafka/external-configuration/elastic-connector-config/connector.properties:truststorePassword}" -``` - -Apply the yaml using: -``` -oc apply -f connector-elastic.yaml -``` - -Find the connector pod and watch the logs: -``` -CONNECTOR_POD=$(oc get pod -l eventstreams.ibm.com/cluster=eei-cluster --output=jsonpath={.items..metadata.name}) -echo "CONNECTOR_POD=${CONNECTOR_POD}" -oc logs -f $CONNECTOR_POD -``` - -The following should appear in the logs: -``` -2020-10-09 14:12:34,522 INFO Building documents using com.ibm.eventstreams.connect.elasticsink.builders.JsonDocumentBuilder (com.ibm.eventstreams.connect.elasticsink.builders.JsonDocumentBuilder) [task-thread-eei-elastic-0] -2020-10-09 14:12:35,075 INFO Connection to Elasticsearch established (com.ibm.eventstreams.connect.elasticsink.ElasticWriter) [task-thread-eei-elastic-0] -2020-10-09 14:12:35,076 INFO WorkerSinkTask{id=eei-elastic-0} Sink task finished initialization and start (org.apache.kafka.connect.runtime.WorkerSinkTask) [task-thread-eei-elastic-0] -2020-10-09 14:12:38,853 INFO WorkerSinkTask{id=eei-elastic-0} Committing offsets asynchronously using sequence number 1: {sor.public.quotes-0=OffsetAndMetadata{offset=65, leaderEpoch=null, metadata=''}} (org.apache.kafka.connect.runtime.WorkerSinkTask) [task-thread-eei-elastic-0] -``` -And now the connector is monitoring the `sor.public.quotes` topic and writing to the `sor.public.quotes` index in Elasticsearch. - -# Verify contents of Elasticsearch -Port forward the Elasticsearch service to your localhost: -``` -ELASTIC_NAMESPACE=cp4i -oc port-forward -n ${ELASTIC_NAMESPACE} service/elasticsearch-eei-es-http 9200 -``` - -In a separate terminal setup some env vars to allow curl to call Elasticsearch: -``` -ELASTIC_NAMESPACE=cp4i -ELASTIC_PASSWORD=$(oc get secret elasticsearch-eei-es-elastic-user -n $ELASTIC_NAMESPACE -o go-template='{{.data.elastic | base64decode}}') -ELASTIC_USER="elastic" -``` - -Check that the `sor.public.quotes` index has been created using the command: -``` -curl -u "${ELASTIC_USER}:${ELASTIC_PASSWORD}" -k 'https://localhost:9200/_cat/indices?v' -``` -Which should return something like: -``` -health status index uuid pri rep docs.count docs.deleted store.size pri.store.size -yellow open sor.public.quotes fb59dXjURR6uCxRwoRpaIQ 1 1 18 47 39.7kb 39.7kb -``` - -Get the current state of the `sor.public.quotes` index into an env var named `JSON`: -``` -JSON=$(curl -u "${ELASTIC_USER}:${ELASTIC_PASSWORD}" -k "https://localhost:9200/sor.public.quotes/_search" -H 'Content-Type: application/json' -d'{"query":{"match_all":{}}}') -``` - -Get the number of rows from the JSON using: -``` -echo $JSON | jq .hits.total.value -``` - -Get the rows from the JSON using: -``` -echo $JSON | jq '.hits.hits[]._source.after' -``` -Which should return something like: -``` -{ - "quoteid": "22d3ae0a-3498-4207-ae99-b7fe584f6c1b", - "source": "Mobile", - "name": "Nella Beard", - "email": "NBeard@mail.com", - "age": 45, - "address": "8774 Inverness Dr., Janesville", - "usstate": "WI", - "licenseplate": "787-YWR", - "descriptionofdamage": "Wheel fell off", - "claimstatus": 2, - "claimcost": null -} -{ - "quoteid": "12abfe16-0c41-42a4-9edb-201f79ef05c2", - "source": "Mobile", - "name": "Andy Rosales", - "email": "AndyR@mail.com", - "age": 77, - "address": "9783 Oxford St., Duluth", - "usstate": "GA", - "licenseplate": "GWL3149", - "descriptionofdamage": "Won't start", - "claimstatus": 3, - "claimcost": null -} -... -``` - -To delete the index: -``` -curl -X DELETE -u "${ELASTIC_USER}:${ELASTIC_PASSWORD}" -k https://localhost:9200/sor.public.quotes -``` - # Working directly with the System Of Record database Setup some env vars ``` diff --git a/EventEnabledInsurance/setup-elastic-search.sh b/EventEnabledInsurance/setup-elastic-search.sh deleted file mode 100755 index 1b14deb84..000000000 --- a/EventEnabledInsurance/setup-elastic-search.sh +++ /dev/null @@ -1,287 +0,0 @@ -#!/bin/bash -#****************************************************************************** -# Licensed Materials - Property of IBM -# (c) Copyright IBM Corporation 2019. All Rights Reserved. -# -# Note to U.S. Government Users Restricted Rights: -# Use, duplication or disclosure restricted by GSA ADP Schedule -# Contract with IBM Corp. -#****************************************************************************** - -# PREREQUISITES: -# - Logged into cluster on the OC CLI (https://docs.openshift.com/container-platform/4.4/cli_reference/openshift_cli/getting-started-cli.html) -# -# PARAMETERS: -# -e : (string), Defaults to 'elasticsearch', the namespace where elastic search is installed to -# -n : (string), Defaults to 'cp4i' -# -# With defaults values -# ./setup-elastic-search.sh -# -# With overridden values -# ./setup-elastic-search.sh -n -e -# - -function usage() { - echo "Usage: $0 -n -e " - exit 1 -} - -tick="\xE2\x9C\x85" -cross="\xE2\x9D\x8C" -all_done="\xF0\x9F\x92\xAF" -SUFFIX="eei" -ELASTIC_CR_NAME="elasticsearch-$SUFFIX" -NAMESPACE="cp4i" -ELASTIC_NAMESPACE="elasticsearch" -ELASTIC_SUBSCRIPTION_NAME="elasticsearch-eck-operator-certified" - -while getopts "n:e:" opt; do - case ${opt} in - n) - NAMESPACE="$OPTARG" - ;; - e) - ELASTIC_NAMESPACE="$OPTARG" - ;; - \?) - usage - ;; - esac -done - -if [[ -z "${NAMESPACE// /}" || -z "${ELASTIC_NAMESPACE// /}" ]]; then - echo -e "$cross ERROR: Mandatory parameters are missing" - usage -fi - -echo -e "\n----------------------------------------------------------------------------------------------------------------------------------------------------------\n" - -CURRENT_DIR=$(dirname $0) -echo "INFO: Current directory: '$CURRENT_DIR'" -echo "INFO: Elastic Namespace: '$ELASTIC_NAMESPACE'" -echo "INFO: Elastic search CR name: '$ELASTIC_CR_NAME'" -echo "INFO: Namespace: $NAMESPACE" -echo "INFO: Elastic search subscription: '$ELASTIC_SUBSCRIPTION_NAME'" -echo "INFO: Suffix is: '$SUFFIX'" - -echo -e "\n----------------------------------------------------------------------------------------------------------------------------------------------------------\n" - -oc create namespace $ELASTIC_NAMESPACE - -echo -e "\n----------------------------------------------------------------------------------------------------------------------------------------------------------\n" - -function output_time() { - SECONDS=${1} - if ((SECONDS > 59)); then - printf "%d minutes, %d seconds" $((SECONDS / 60)) $((SECONDS % 60)) - else - printf "%d seconds" $SECONDS - fi -} - -function wait_for_subscription() { - ELASTIC_NAMESPACE=${1} - NAME=${2} - phase="" - time=0 - wait_time=5 - until [[ "$phase" == "Succeeded" ]]; do - csv=$(oc get subscription -n ${ELASTIC_NAMESPACE} ${NAME} -o json | jq -r .status.currentCSV) - wait=0 - if [[ "$csv" == "null" ]]; then - echo "Waited for $(output_time $time), not got csv for subscription $NAME" - wait=1 - else - phase=$(oc get csv -n ${ELASTIC_NAMESPACE} $csv -o json | jq -r .status.phase) - if [[ "$phase" != "Succeeded" ]]; then - echo "Waited for $(output_time $time), csv $csv not in Succeeded phase, currently: $phase" - wait=1 - fi - fi - - if [[ "$wait" == "1" ]]; then - ((time = time + $wait_time)) - if [ $time -gt 1200 ]; then - echo "ERROR: Failed after waiting for 20 minutes for $NAME" - exit 1 - fi - - sleep $wait_time - fi - done - echo "$NAME has succeeded" -} - -OPERATOR_GROUP_COUNT=$(oc get operatorgroups -n ${ELASTIC_NAMESPACE} -o json | jq '.items | length') -if [[ "${OPERATOR_GROUP_COUNT}" == "0" ]]; then - cat < /dev/null) -if [[ $? == 0 ]]; then - METADATA_NAME=$(echo $json | tr '\r\n' ' ' | jq -r '.data.METADATA_NAME') - METADATA_UID=$(echo $json | tr '\r\n' ' ' | jq -r '.data.METADATA_UID') -fi - -cat </tmp/ca.crt -rm /tmp/elastic-ts.jks - -TRUSTSTORE_PASSWORD=$( - LC_ALL=C tr -dc 'A-Za-z0-9' /dev/null) -if [[ $? == 0 ]]; then - METADATA_NAME=$(echo $json | tr '\r\n' ' ' | jq -r '.data.METADATA_NAME') - METADATA_UID=$(echo $json | tr '\r\n' ' ' | jq -r '.data.METADATA_UID') -fi - -SECRET_EXISTS=$(oc -n $NAMESPACE get secret eei-elastic-credential) -if [[ -z $SECRET_EXISTS ]]; then - cat <