From 1394e66d689b720de3159eded48581ffc4392c71 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Mon, 20 Nov 2023 12:17:51 +0530 Subject: [PATCH 01/46] #OB-553 fix: Deployment issue fixes for OCI --- .../analytics-spark-provision/tasks/main.yml | 4 +-- .../data-products-deploy/defaults/main.yml | 3 +- .../roles/data-products-deploy/tasks/main.yml | 32 +++++++++---------- 3 files changed, 20 insertions(+), 19 deletions(-) diff --git a/ansible/roles/analytics-spark-provision/tasks/main.yml b/ansible/roles/analytics-spark-provision/tasks/main.yml index 65731cecb9..b690ba5480 100644 --- a/ansible/roles/analytics-spark-provision/tasks/main.yml +++ b/ansible/roles/analytics-spark-provision/tasks/main.yml @@ -146,14 +146,14 @@ - name: Download Kafka-2.11 become: yes become_user: "{{ analytics_user }}" - get_url: url=http://downloads.mesosphere.com/kafka/assets/kafka_2.11-0.10.1.0.tgz dest={{ analytics.home }}/kafka_2.11-0.10.1.0.tgz force=no owner={{ analytics_user }} group={{ analytics_group }} + get_url: url=https://archive.apache.org/dist/kafka/2.8.0/kafka_2.12-2.8.0.tgz dest={{ analytics.home }}/kafka_2.12-2.8.0.tgz force=no owner={{ analytics_user }} group={{ analytics_group }} tags: - kafka-provision - name: Unarchive Kafka become: yes become_user: "{{ analytics_user }}" - unarchive: src={{ analytics.soft_path }}/kafka_2.11-0.10.1.0.tgz dest={{ analytics.home }}/ copy=no owner={{ analytics_user }} group={{ analytics_group }} creates={{ analytics.home }}/kafka_2.11-0.10.1.0 + unarchive: src={{ analytics.soft_path }}/kafka_2.12-2.8.0.tgz dest={{ analytics.home }}/ copy=no owner={{ analytics_user }} group={{ analytics_group }} creates={{ analytics.home }}/kafka_2.12-2.8.0 tags: - kafka-provision diff --git a/ansible/roles/data-products-deploy/defaults/main.yml b/ansible/roles/data-products-deploy/defaults/main.yml index 06cdf1b0f2..a564f3c66a 100755 --- a/ansible/roles/data-products-deploy/defaults/main.yml +++ b/ansible/roles/data-products-deploy/defaults/main.yml @@ -5,7 +5,8 @@ oci_install_loc: /home/{{analytics_user}}/bin/ bucket: "telemetry-data-store" secor_bucket: "telemetry-data-store" -dp_object_store_type: "azure" +# dp_object_store_type: "azure" +cloud_service_provider: "azure" dp_raw_telemetry_backup_location: "unique/raw/" dp_storage_key_config: "azure_storage_key" dp_storage_secret_config: "azure_storage_secret" diff --git a/ansible/roles/data-products-deploy/tasks/main.yml b/ansible/roles/data-products-deploy/tasks/main.yml index 1c15e14023..6947508627 100644 --- a/ansible/roles/data-products-deploy/tasks/main.yml +++ b/ansible/roles/data-products-deploy/tasks/main.yml @@ -1,20 +1,20 @@ ## Data products deployment ## - name: Ensure azure blob storage container exists command: az storage container create --name {{ bucket }} - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - always - name: Ensure oci oss bucket exists command: "{{oci_install_loc}}/oci os bucket get --name {{ bucket }}" register: check_bucket - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - always - name: Create oci oss bucket command: "{{oci_install_loc}}/oci os bucket create -c {{oci_bucket_compartment}} --name {{bucket}}" - when: dp_object_store_type == "oci" and check_bucket.rc !=0 + when: cloud_service_provider == "oci" and check_bucket.rc !=0 tags: - always @@ -27,7 +27,7 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} async: 3600 poll: 10 - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - dataproducts-spark-cluster @@ -35,7 +35,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_batch_module_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_batch_module_artifact }} --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - dataproducts-spark-cluster @@ -49,7 +49,7 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar -f {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar async: 3600 poll: 10 - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - ed-dataproducts-spark-cluster @@ -57,7 +57,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/data-products-1.0.jar --file {{ analytics.home }}/models-{{ model_version }}/data-products-1.0/data-products-1.0.jar --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - ed-dataproducts-spark-cluster @@ -70,7 +70,7 @@ command: az storage blob upload --overwrite --debug -c {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} async: 3600 poll: 10 - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - framework-spark-cluster @@ -78,7 +78,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ analytics_core_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ analytics_core_artifact }} --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - framework-spark-cluster @@ -91,7 +91,7 @@ command: az storage blob upload --overwrite -c {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} -f {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} async: 3600 poll: 10 - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - framework-spark-cluster @@ -99,7 +99,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/{{ scruid_artifact }} --file {{ analytics.home }}/models-{{ model_version }}/{{ scruid_artifact }} --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - framework-spark-cluster @@ -136,7 +136,7 @@ line: 'spark_output_temp_dir="/var/log/sparkapp/tmp/"' tags: - framework-spark-cluster - when: dp_object_store_type != "oci" + when: cloud_service_provider != "oci" - name: Update spark temp dir value for cluster lineinfile: @@ -145,7 +145,7 @@ line: 'spark_output_temp_dir="/var/log/spark/"' tags: - framework-spark-cluster - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" - name: Update logger kafka config for cluster @@ -160,7 +160,7 @@ command: az storage blob upload --overwrite -c {{ bucket }} -f {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf async: 3600 poll: 10 - when: dp_object_store_type == "azure" + when: cloud_service_provider == "azure" tags: - framework-spark-cluster @@ -168,7 +168,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --file {{ analytics.home }}/models-{{ model_version }}/application.conf --name models-{{ model_version }}/application.conf --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - framework-spark-cluster @@ -416,7 +416,7 @@ command: "{{oci_install_loc}}/oci os object put -bn {{ bucket }} --name models-{{ model_version }}/jets3t.properties --file {{ analytics_cluster.home }}/jets3t.properties --content-type auto --force" async: 3600 poll: 10 - when: dp_object_store_type == "oci" + when: cloud_service_provider == "oci" tags: - replay-job - run-job From a6fe4afebeb1bcb7f4e9ce9f67dfd8d737fc48f7 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Mon, 20 Nov 2023 12:43:28 +0530 Subject: [PATCH 02/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/roles/analytics-druid/defaults/main.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/roles/analytics-druid/defaults/main.yml b/ansible/roles/analytics-druid/defaults/main.yml index 9c160b25b2..6a7e6b8b63 100644 --- a/ansible/roles/analytics-druid/defaults/main.yml +++ b/ansible/roles/analytics-druid/defaults/main.yml @@ -212,10 +212,10 @@ default_druid_configs: druid_log_azure_folder: "druidlogs" #Druid S3 Details druid_storage_type: "{{ cloud_storage_telemetry_type }}" - s3_access_key: "{{ s3_storage_key }}" - s3_secret_key: "{{ s3_storage_secret }}" - s3_bucket: "{{ s3_storage_container }}" - s3_endpoint: "{{ s3_storage_endpoint }}" + s3_access_key: "{{ cloud_private_storage_accountname }}" + s3_secret_key: "{{ cloud_private_storage_secret }}" + s3_bucket: "{{ cloud_storage_telemetry_bucketname }}" + s3_endpoint: "{{ cloud_private_storage_endpoint }}" s3_segment_dir: "druid/rollup/segments" s3_path_like_access: "{{ cloud_storage_pathstyle_access }}" s3_v4_sign_region: "{{ cloud_private_storage_region }}" From 16fe19c097f7b852bb714e6d3ee36194798fd1f8 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Mon, 20 Nov 2023 14:32:46 +0530 Subject: [PATCH 03/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/roles/data-products-deploy/templates/submit-job.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/data-products-deploy/templates/submit-job.j2 b/ansible/roles/data-products-deploy/templates/submit-job.j2 index 859cf602c3..6e4052f68e 100644 --- a/ansible/roles/data-products-deploy/templates/submit-job.j2 +++ b/ansible/roles/data-products-deploy/templates/submit-job.j2 @@ -3,7 +3,7 @@ export SPARK_HOME={{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7 export MODELS_HOME={{ analytics.home }}/models-{{ model_version }} export DP_LOGS={{ analytics.home }}/logs/data-products -export KAFKA_HOME={{ analytics.soft_path }}/kafka_2.11-0.10.1.0 +export KAFKA_HOME={{ analytics.soft_path }}/kafka_2.12-2.8.0 ## job broker-list and kafka-topic job_brokerList={{ brokerlist }} From f6a9ce26258e50a0453c28ef5c6869af029e2ab3 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Mon, 20 Nov 2023 15:16:04 +0530 Subject: [PATCH 04/46] #OB-553 fix: Deployment issue fixes for OCI --- .../data-products-deploy/templates/cluster-config.json.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 b/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 index a5fb617373..25d6703c57 100644 --- a/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 +++ b/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 @@ -76,8 +76,8 @@ "spark.cassandra.connection.timeoutMS" : "{{ spark_cassandra_connection_timeout_millis }}", "spark.cassandra.read.timeoutMS" : "{{ spark_cassandra_query_timeout_millis }}", "spark.cassandra.input.fetch.sizeInRows": "{{ spark_cassandra_query_max_rows_fetch_count }}", - "spark.driver.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/spark3/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/odh/2.0.1/spark/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{sunbird_private_storage_account_name}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}}, -Daws_storage_key={{ sunbird_private_storage_account_name }} -Daws_storage_secret={{ sunbird_private_storage_account_key }} -Dcom.amazonaws.sdk.s3.defaultStreamBufferSize=512m", - "spark.executor.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/spark3/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/odh/2.0.1/spark/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ sunbird_private_storage_account_name }} -Dazure_storage_secret={{ sunbird_private_storage_account_key }} -Dreports_storage_key={{sunbird_private_storage_account_name}} -Dreports_storage_secret={{sunbird_private_storage_account_key}} -Ddruid_storage_account_key={{ sunbird_public_storage_account_name }} -Ddruid_storage_account_secret={{sunbird_public_storage_account_key}} -Daws_storage_key={{ sunbird_private_storage_account_name }} -Daws_storage_secret={{ sunbird_private_storage_account_key }} -Dcom.amazonaws.sdk.s3.defaultStreamBufferSize=512m" + "spark.driver.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/spark3/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/odh/2.0.1/spark/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ cloud_private_storage_accountname }} -Dazure_storage_secret={{ cloud_private_storage_secret }} -Dreports_storage_key={{cloud_private_storage_accountname}} -Dreports_storage_secret={{cloud_private_storage_secret}} -Ddruid_storage_account_key={{ cloud_public_storage_accountname }} -Ddruid_storage_account_secret={{cloud_public_storage_secret}}, -Daws_storage_key={{ cloud_private_storage_accountname }} -Daws_storage_secret={{ cloud_private_storage_secret }} -Dcom.amazonaws.sdk.s3.defaultStreamBufferSize=512m", + "spark.executor.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/spark3/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/odh/2.0.1/spark/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ cloud_private_storage_accountname }} -Dazure_storage_secret={{ cloud_private_storage_secret }} -Dreports_storage_key={{cloud_private_storage_accountname}} -Dreports_storage_secret={{cloud_private_storage_secret}} -Ddruid_storage_account_key={{ cloud_public_storage_accountname }} -Ddruid_storage_account_secret={{cloud_public_storage_secret}} -Daws_storage_key={{ cloud_private_storage_accountname }} -Daws_storage_secret={{ cloud_private_storage_secret }} -Dcom.amazonaws.sdk.s3.defaultStreamBufferSize=512m" } } {% endif %} \ No newline at end of file From 03a8bd51f5e4b42592dd73a8084480f694153ea2 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 21 Nov 2023 10:45:24 +0530 Subject: [PATCH 05/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/lpa_data-products_deploy.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/lpa_data-products_deploy.yml b/ansible/lpa_data-products_deploy.yml index 903afcc7b3..b37d04b177 100644 --- a/ansible/lpa_data-products_deploy.yml +++ b/ansible/lpa_data-products_deploy.yml @@ -7,5 +7,6 @@ environment: AZURE_STORAGE_ACCOUNT: "{{cloud_private_storage_accountname}}" AZURE_STORAGE_KEY: "{{cloud_private_storage_secret}}" + OCI_CLI_AUTH: "instance_principal" roles: - data-products-deploy From 6f6213ed5632153085d8529748b8d24b957f476d Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 21 Nov 2023 10:56:25 +0530 Subject: [PATCH 06/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/roles/analytics-druid/defaults/main.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/roles/analytics-druid/defaults/main.yml b/ansible/roles/analytics-druid/defaults/main.yml index 6a7e6b8b63..189f4c1d7e 100644 --- a/ansible/roles/analytics-druid/defaults/main.yml +++ b/ansible/roles/analytics-druid/defaults/main.yml @@ -30,16 +30,16 @@ druid_request_logging_type: "file" #Druid Extensions -druid_storage_type: "{{ cloud_storage_telemetry_type }}" +druid_storage_type: "{{ druid_storage_type }}" -druid_extensions_list: '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches"' +druid_extensions_list: '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-s3-extensions"' druid_community_extensions: - graphite-emitter # End of druid_extensions -druid_indexing_logs_type: "{{ cloud_storage_telemetry_type }}" +druid_indexing_logs_type: "{{ druid_storage_type }}" druid_indexing_log_dir: /var/druid/indexing-logs druid_indexing_storage_type: metadata druid_indexing_task_basedir: "/var/task" From fda6588d3ca43e1211d364f43bb15296cf24f46c Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 21 Nov 2023 11:24:18 +0530 Subject: [PATCH 07/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/roles/analytics-druid/defaults/main.yml | 2 +- .../templates/_common/common.runtime.properties | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/ansible/roles/analytics-druid/defaults/main.yml b/ansible/roles/analytics-druid/defaults/main.yml index 189f4c1d7e..dbbb714681 100644 --- a/ansible/roles/analytics-druid/defaults/main.yml +++ b/ansible/roles/analytics-druid/defaults/main.yml @@ -32,7 +32,7 @@ druid_request_logging_type: "file" druid_storage_type: "{{ druid_storage_type }}" -druid_extensions_list: '"druid-azure-extensions", "graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches", "druid-s3-extensions"' +druid_default_extensions_list: '"graphite-emitter", "postgresql-metadata-storage", "druid-kafka-indexing-service", "druid-datasketches"' druid_community_extensions: - graphite-emitter diff --git a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties index be217e65c8..7a3e44c9e6 100644 --- a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties +++ b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties @@ -24,7 +24,10 @@ # If you specify `druid.extensions.loadList=[]`, Druid won't load any extension from file system. # If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. # More info: http://druid.io/docs/latest/operations/including-extensions.html -druid.extensions.loadList=[{{ druid_extensions_list }}] +{% if druid_storage_type == "azure" %} +druid.extensions.loadList=[{{ druid_default_extensions_list }}, "druid-azure-extensions] +{% elif druid_storage_type == "s3" %} +druid.extensions.loadList=[{{ druid_default_extensions_list }}, "druid-s3-extensions] druid.extensions.directory={{ druid_path }}extensions # Logging From 56661bff94240e4de419f3313ead95eaa7e923f0 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 21 Nov 2023 11:44:57 +0530 Subject: [PATCH 08/46] #OB-553 fix: Deployment issue fixes for OCI --- .../analytics-druid/templates/_common/common.runtime.properties | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties index 7a3e44c9e6..3a573ff6b7 100644 --- a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties +++ b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties @@ -28,6 +28,7 @@ druid.extensions.loadList=[{{ druid_default_extensions_list }}, "druid-azure-extensions] {% elif druid_storage_type == "s3" %} druid.extensions.loadList=[{{ druid_default_extensions_list }}, "druid-s3-extensions] +{% endif %} druid.extensions.directory={{ druid_path }}extensions # Logging From 661b05a91d4e0fc1b562b783427ebf7d64ae153b Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 21 Nov 2023 12:46:12 +0530 Subject: [PATCH 09/46] #OB-553 fix: Deployment issue fixes for OCI --- .../templates/_common/common.runtime.properties | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties index 3a573ff6b7..d78de55caf 100644 --- a/ansible/roles/analytics-druid/templates/_common/common.runtime.properties +++ b/ansible/roles/analytics-druid/templates/_common/common.runtime.properties @@ -25,9 +25,9 @@ # If you don't specify `druid.extensions.loadList`, Druid will load all the extensions under root extension directory. # More info: http://druid.io/docs/latest/operations/including-extensions.html {% if druid_storage_type == "azure" %} -druid.extensions.loadList=[{{ druid_default_extensions_list }}, "druid-azure-extensions] +druid.extensions.loadList=[{{ druid_default_extensions_list }}, "druid-azure-extensions"] {% elif druid_storage_type == "s3" %} -druid.extensions.loadList=[{{ druid_default_extensions_list }}, "druid-s3-extensions] +druid.extensions.loadList=[{{ druid_default_extensions_list }}, "druid-s3-extensions"] {% endif %} druid.extensions.directory={{ druid_path }}extensions From 01858fa726194769bfe153955d28aa063498a7ab Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 21 Nov 2023 13:45:36 +0530 Subject: [PATCH 10/46] #OB-553 fix: Deployment issue fixes for OCI --- .../templates/cluster-config.json.j2 | 4 +- .../collection-summary-ingestion-spec.j2 | 4 +- .../templates/common.conf.j2 | 8 +-- .../templates/model-config.j2 | 60 +++++++++---------- .../templates/model-config.json.j2 | 40 ++++++------- .../templates/model-dock-config.j2 | 8 +-- .../templates/sourcing-ingestion-spec.j2 | 4 +- .../templates/submit-script.j2 | 16 ++--- 8 files changed, 72 insertions(+), 72 deletions(-) diff --git a/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 b/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 index 25d6703c57..8f029b8b3f 100644 --- a/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 +++ b/ansible/roles/data-products-deploy/templates/cluster-config.json.j2 @@ -1,5 +1,5 @@ -{% if dp_object_store_type == "azure" %} +{% if cloud_service_provider == "azure" %} { "jars": [ {% if cloud_storage_telemetry_type == "azure" %} @@ -50,7 +50,7 @@ "spark.executor.extraJavaOptions": "-Detwlogger.component=sparkdriver -DlogFilter.filename=SparkLogFilters.xml -DpatternGroup.filename=SparkPatternGroups.xml -Dlog4jspark.root.logger=INFO,console,RFA,ETW,Anonymizer,org.ekstep.analytics -Dlog4jspark.log.dir=/var/log/sparkapp/${user.name} -Dlog4jspark.log.file=sparkdriver.log -Dlog4j.configuration=file:/usr/hdp/current/spark2-client/conf/log4j.properties -Djavax.xml.parsers.SAXParserFactory=com.sun.org.apache.xerces.internal.jaxp.SAXParserFactoryImpl -XX:+UseParallelGC -XX:+UseParallelOldGC -Dazure_storage_key={{ cloud_private_storage_accountname }} -Dazure_storage_secret={{ cloud_private_storage_secret }} -Dreports_storage_key={{cloud_private_storage_accountname}} -Dreports_storage_secret={{cloud_private_storage_secret}} -Ddruid_storage_account_key={{ cloud_public_storage_accountname }} -Ddruid_storage_account_secret={{cloud_public_storage_secret}}" } } -{% elif (dp_object_store_type == "oci") %} +{% elif (cloud_service_provider == "oci") %} { "jars": [ "oci://{{ bucket }}@{{ oci_namespace }}/models-{{ model_version }}/{{ analytics_core_artifact }}", diff --git a/ansible/roles/data-products-deploy/templates/collection-summary-ingestion-spec.j2 b/ansible/roles/data-products-deploy/templates/collection-summary-ingestion-spec.j2 index f26c2e6447..100724ced7 100644 --- a/ansible/roles/data-products-deploy/templates/collection-summary-ingestion-spec.j2 +++ b/ansible/roles/data-products-deploy/templates/collection-summary-ingestion-spec.j2 @@ -230,7 +230,7 @@ "ioConfig": { "type": "index", "firehose": { -{% if dp_object_store_type == "azure" %} +{% if cloud_service_provider == "azure" %} "type": "static-azure-blobstore", "blobs": [ { @@ -238,7 +238,7 @@ "path": "/collection-summary-reports-v2/collection-summary-report-latest.json" } ], -{% elif (dp_object_store_type == "oci") %} +{% elif (cloud_service_provider == "oci") %} "type": "static-s3", "uris": [ "s3://{{reports_container}}/sourcing/SourcingSummaryReport.json"], {% endif %} diff --git a/ansible/roles/data-products-deploy/templates/common.conf.j2 b/ansible/roles/data-products-deploy/templates/common.conf.j2 index d29f854f8f..ae013008aa 100644 --- a/ansible/roles/data-products-deploy/templates/common.conf.j2 +++ b/ansible/roles/data-products-deploy/templates/common.conf.j2 @@ -12,9 +12,9 @@ storage.key.config="{{ dp_storage_key_config }}" storage.secret.config="{{ dp_storage_secret_config }}" reports.storage.key.config="{{ dp_reports_storage_key_config }}" reports.storage.secret.config="{{ dp_reports_storage_secret_config }}" -{% if dp_object_store_type == "azure" %} +{% if cloud_service_provider == "azure" %} cloud_storage_telemetry_type="azure" -{% elif (dp_object_store_type == "cephs3" or dp_object_store_type == "s3") %} +{% elif (cloud_service_provider == "cephs3" or cloud_service_provider == "s3") %} cloud_storage_telemetry_type="s3" cloud_private_storage_endpoint="{{ s3_storage_endpoint | regex_replace('^[a-z]+://(.*)$', '\\1') }}" cloud_storage_endpoint_with_protocol="{{ s3_storage_endpoint }}" @@ -249,9 +249,9 @@ dcetextbook.filename="DCE_textbook_data.csv" etbtextbook.filename="ETB_textbook_data.csv" etb.dialcode.druid.length={{ etb_dialcode_list_druid_length }} -{% if dp_object_store_type == "azure" %} +{% if cloud_service_provider == "azure" %} druid.report.default.storage="azure" -{% elif (dp_object_store_type == "oci" or dp_object_store_type == "s3") %} +{% elif (cloud_service_provider == "oci" or cloud_service_provider == "s3") %} druid.report.default.storage="s3" {% endif %} diff --git a/ansible/roles/data-products-deploy/templates/model-config.j2 b/ansible/roles/data-products-deploy/templates/model-config.j2 index f9991e1714..5a7f3d2c4d 100644 --- a/ansible/roles/data-products-deploy/templates/model-config.j2 +++ b/ansible/roles/data-products-deploy/templates/model-config.j2 @@ -30,24 +30,24 @@ config() { if [ ! -z "$2" ]; then keyword=$2; fi case "$1" in "assessment-correction") - echo '{"search":{"type":"{{dp_object_store_type}}","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"model":"org.sunbird.analytics.model.report.AssessmentCorrectionModel","modelParams":{"parallelization":200,"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"}],"filters":[{"type":"equals","dimension":"contentType","value":"SelfAssess"}],"descending":"false"},"fileOutputConfig":{"to":"file","params":{"file":"{{ analytics.home }}/assessment-correction/skippedEvents"}},"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"kafka","params":{"brokerList":"'$brokerIngestionList'","topic":"'$assessTopic'"}}],"parallelization":200,"appName":"Assessment Correction Model"}' + echo '{"search":{"type":"{{cloud_service_provider}}","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"model":"org.sunbird.analytics.model.report.AssessmentCorrectionModel","modelParams":{"parallelization":200,"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"}],"filters":[{"type":"equals","dimension":"contentType","value":"SelfAssess"}],"descending":"false"},"fileOutputConfig":{"to":"file","params":{"file":"{{ analytics.home }}/assessment-correction/skippedEvents"}},"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"kafka","params":{"brokerList":"'$brokerIngestionList'","topic":"'$assessTopic'"}}],"parallelization":200,"appName":"Assessment Correction Model"}' ;; "assessment-archival") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"store":"{{ dp_object_store_type }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Job"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Job"}' ;; "assessment-archived-removal") -{% if dp_object_store_type == "azure" %} +{% if cloud_service_provider == "azure" %} echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"deleteArchivedBatch":true,"azureFetcherConfig":{"store":"azure","blobExt":"csv.gz","reportPath":"archived-data/","container":"reports"},"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Removal Job"}' ;; -{% elif (dp_object_store_type == "oci" or dp_object_store_type == "s3") %} - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"deleteArchivedBatch":true,"azureFetcherConfig":{"store":"{{dp_object_store_type}}","blobExt":"csv.gz","reportPath":"archived-data/","container":"{{reports_container}}"},"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Removal Job"}' +{% elif (cloud_service_provider == "oci" or cloud_service_provider == "s3") %} + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"deleteArchivedBatch":true,"azureFetcherConfig":{"store":"{{cloud_service_provider}}","blobExt":"csv.gz","reportPath":"archived-data/","container":"{{reports_container}}"},"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Removal Job"}' ;; {% endif %} "collection-reconciliation-job") echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CollectionReconciliationJob","modelParams":{"mode":"prodrun","brokerList":"{{ingestion_kafka_broker_host}}","topic":"{{env}}.issue.certificate.request","sparkCassandraConnectionHost":"{{ core_cassandra_host }}"},"parallelization":30,"appName":"CollectionReconciliationJob"}' ;; "collection-summary-report") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course"},"fields":["identifier","name","organisation","channel"],"limit":10000}},"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course"},"fields":["identifier","name","organisation","channel"],"limit":10000}},"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' ;; "score-metric-migration-job") echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.ScoreMetricMigrationJob","modelParams":{"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Score Metric Migration Job"}' @@ -56,34 +56,34 @@ config() { echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.AssessmentScoreCorrectionJob","modelParams":{"assessment.score.correction.batches":"","cassandraReadConsistency":"QUORUM","cassandraWriteConsistency":"QUORUM","csvPath":"/mount/data/analytics/score_correction","isDryRunMode":true,"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":30,"appName":"Assessment Score Correction Job"}' ;; "course-batch-status-updater") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CourseBatchStatusUpdaterJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"http://{{ single_node_es_host }}:9200","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","kpLearningBasePath":"http://{{groups['learning'][0]}}:8080/learning-service","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Course Batch Status Updater Job"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CourseBatchStatusUpdaterJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"http://{{ single_node_es_host }}:9200","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","kpLearningBasePath":"http://{{groups['learning'][0]}}:8080/learning-service","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Course Batch Status Updater Job"}' ;; "collection-summary-report-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJobV2","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","batchSize":50,"generateForAllBatches":true,"contentFields":["identifier","name","organisation","channel","status","keywords","createdFor","medium","subject"],"contentStatus":["Live","Unlisted","Retired"],"store":"{{dp_object_store_type}}","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","specPath":"/mount/data/analytics/scripts/collection-summary-ingestion-spec.json","druidIngestionUrl":"'$druidIngestionURL'","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report V2"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJobV2","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","batchSize":50,"generateForAllBatches":true,"contentFields":["identifier","name","organisation","channel","status","keywords","createdFor","medium","subject"],"contentStatus":["Live","Unlisted","Retired"],"store":"{{cloud_service_provider}}","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","specPath":"/mount/data/analytics/scripts/collection-summary-ingestion-spec.json","druidIngestionUrl":"'$druidIngestionURL'","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report V2"}' ;; "uci-private-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.uci.UCIPrivateExhaustJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UCI Private Exhaust"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.uci.UCIPrivateExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UCI Private Exhaust"}' ;; "uci-response-exhaust") - echo '{"search":{"type":"{{dp_object_store_type}}","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"filters":[{"name":"eid","operator":"EQ","value":"ASSESS"}],"model":"org.sunbird.analytics.uci.UCIResponseExhaust","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","botPdataId":"{{ uci_pdata_id }}","mode":"OnDemand","fromDate":"","toDate":""},"parallelization":8,"appName":"UCI Response Exhaust"}' + echo '{"search":{"type":"{{cloud_service_provider}}","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"filters":[{"name":"eid","operator":"EQ","value":"ASSESS"}],"model":"org.sunbird.analytics.uci.UCIResponseExhaust","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","botPdataId":"{{ uci_pdata_id }}","mode":"OnDemand","fromDate":"","toDate":""},"parallelization":8,"appName":"UCI Response Exhaust"}' ;; "userinfo-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UserInfo Exhaust"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UserInfo Exhaust"}' ;; "program-collection-summary-report") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course","keywords":["'$keyword'"]},"fields":["identifier","name","organisation","channel"],"limit":10000}},"columns":["Published by","Batch id","Collection id","Collection name","Batch start date","Batch end date","State","Total enrolments By State","Total completion By State"],"keywords":"'$keyword'","store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course","keywords":["'$keyword'"]},"fields":["identifier","name","organisation","channel"],"limit":10000}},"columns":["Published by","Batch id","Collection id","Collection name","Batch start date","Batch end date","State","Total enrolments By State","Total completion By State"],"keywords":"'$keyword'","store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' ;; "response-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust"}' ;; "response-exhaust-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust V2"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust V2"}' ;; "progress-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}", "apiVersion":"v2", "parallelization":200,"mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}", "apiVersion":"v2", "parallelization":200,"mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust"}' ;; "progress-exhaust-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2","modelParams":{"store":"{{dp_object_store_type}}","mode":"OnDemand","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust V2"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2","modelParams":{"store":"{{cloud_service_provider}}","mode":"OnDemand","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust V2"}' ;; "druid_reports") echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"mode":"batch"},"parallelization":8,"appName":"Druid Reports"}' @@ -98,23 +98,23 @@ config() { echo '{"jobsCount":'$jobManagerJobsCount',"topic":"'$job_topic'","bootStrapServer":"'$brokerList'","zookeeperConnect":"'$zookeeper'","consumerGroup":"jobmanager","slackChannel":"#test_channel","slackUserName":"JobManager","tempBucket":"'$bucket'","tempFolder":"'$temp_folder'"}' ;; "wfs") - echo '{"search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"{{ dp_raw_telemetry_backup_location }}","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","store":"{{ dp_object_store_type }}","apiVersion":"v2","parallelization":200},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":200,"appName":"Workflow Summarizer","deviceMapping":true}' - #echo '{"search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"unique/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":8,"appName":"Workflow Summarizer","deviceMapping":true}' + echo '{"search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"{{ dp_raw_telemetry_backup_location }}","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","store":"{{ cloud_service_provider }}","apiVersion":"v2","parallelization":200},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":200,"appName":"Workflow Summarizer","deviceMapping":true}' + #echo '{"search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"unique/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":8,"appName":"Workflow Summarizer","deviceMapping":true}' ;; "video-streaming") - echo '{"search":{"type":"{{ dp_object_store_type }}"},"model":"org.ekstep.analytics.job.VideoStreamingJob","modelParams":{"maxIterations":10},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Video Streaming Job","deviceMapping":false}' + echo '{"search":{"type":"{{ cloud_service_provider }}"},"model":"org.ekstep.analytics.job.VideoStreamingJob","modelParams":{"maxIterations":10},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Video Streaming Job","deviceMapping":false}' ;; "admin-user-reports") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminReportJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin User Reports","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminReportJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin User Reports","deviceMapping":false}' ;; "admin-geo-reports") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminGeoReportJob","modelParams":{"store":"{{dp_object_store_type}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin Geo Reports","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminGeoReportJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin Geo Reports","deviceMapping":false}' ;; "telemetry-replay") - echo '{"search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"'$inputBucket'","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"TelemetryReplayJob","deviceMapping":false}' + echo '{"search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"'$inputBucket'","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"TelemetryReplayJob","deviceMapping":false}' ;; "summary-replay") - echo '{"search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"SummaryReplayJob","deviceMapping":false}' + echo '{"search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"SummaryReplayJob","deviceMapping":false}' ;; "content-rating-updater") echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.updater.UpdateContentRating","modelParams": {"startDate": "'$endDate'","endDate": "'$endDate'"},"output": [{"to":"console","params":{"printEvent":false}}],"parallelization": 8,"appName": "Content Rating Updater","deviceMapping": false}' @@ -123,25 +123,25 @@ config() { echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.ExperimentDefinitionModel","modelParams":{"sparkElasticsearchConnectionHost":"{{ lp_composite_search_host }}"},"output":[{"to":"elasticsearch","params":{"index":"experiment"}}],"parallelization":8,"appName":"Experiment-Definition","deviceMapping":false}' ;; "etb-metrics") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.ETBMetricsJob","modelParams":{"reportConfig":{"id":"etb_metrics","metrics":[],"labels":{"date":"Date","identifier":"Textbook ID","name":"Textbook Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","totalQRCodes":"Total number of QR codes","contentLinkedQR":"Number of QR codes with atleast 1 linked content","withoutContentQR":"Number of QR codes with no linked content","withoutContentT1":"Term 1 QR Codes with no linked content","withoutContentT2":"Term 2 QR Codes with no linked content","status":"Textbook Status","totalContentLinked":"Total content linked","totalQRLinked":"Total QR codes linked to content","totalQRNotLinked":"Total number of QR codes with no linked content","leafNodesCount":"Total number of leaf nodes","leafNodeUnlinked":"Number of leaf nodes with no content","l1Name":"Level 1 Name","l2Name":"Level 2 Name","l3Name":"Level 3 Name","l4Name":"Level 4 Name","l5Name":"Level 5 Name","dialcode":"QR Code","sum(scans)":"Total Scans","noOfContent":"Number of contents","nodeType":"Type of Node","term":"Term"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"dialcode_counts.csv","postContainer":"'$reportPostContainer'"}},"dialcodeReportConfig":{"id":"etb_metrics","metrics":[],"labels":{},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":1,"reportPath":"dialcode_counts.csv","rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":10,"postContainer":"'$reportPostContainer'"}},"etbFileConfig":{"bucket":"'$reportPostContainer'","file":"dialcode_scans/dialcode_counts.csv"},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"equals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live","Draft","Review"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"tenantConfig":{"tenantId":"","slugName":""},"store":"{{ dp_object_store_type }}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ETB Metrics Model","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.ETBMetricsJob","modelParams":{"reportConfig":{"id":"etb_metrics","metrics":[],"labels":{"date":"Date","identifier":"Textbook ID","name":"Textbook Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","totalQRCodes":"Total number of QR codes","contentLinkedQR":"Number of QR codes with atleast 1 linked content","withoutContentQR":"Number of QR codes with no linked content","withoutContentT1":"Term 1 QR Codes with no linked content","withoutContentT2":"Term 2 QR Codes with no linked content","status":"Textbook Status","totalContentLinked":"Total content linked","totalQRLinked":"Total QR codes linked to content","totalQRNotLinked":"Total number of QR codes with no linked content","leafNodesCount":"Total number of leaf nodes","leafNodeUnlinked":"Number of leaf nodes with no content","l1Name":"Level 1 Name","l2Name":"Level 2 Name","l3Name":"Level 3 Name","l4Name":"Level 4 Name","l5Name":"Level 5 Name","dialcode":"QR Code","sum(scans)":"Total Scans","noOfContent":"Number of contents","nodeType":"Type of Node","term":"Term"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"dialcode_counts.csv","postContainer":"'$reportPostContainer'"}},"dialcodeReportConfig":{"id":"etb_metrics","metrics":[],"labels":{},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":1,"reportPath":"dialcode_counts.csv","rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":10,"postContainer":"'$reportPostContainer'"}},"etbFileConfig":{"bucket":"'$reportPostContainer'","file":"dialcode_scans/dialcode_counts.csv"},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"equals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live","Draft","Review"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"tenantConfig":{"tenantId":"","slugName":""},"store":"{{ cloud_service_provider }}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ETB Metrics Model","deviceMapping":false}' ;; "course-enrollment-report") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseEnrollmentJob","modelParams":{"reportConfig":{"id":"tpd_metrics","metrics":[],"labels":{"completionCount":"Completion Count","status":"Status","enrollmentCount":"Enrollment Count","courseName":"Course Name","batchName":"Batch Name"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"course_enrollment.csv"}},"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]},"limit":10000}},"store":"{{ dp_object_store_type }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Enrollment Metrics Model","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseEnrollmentJob","modelParams":{"reportConfig":{"id":"tpd_metrics","metrics":[],"labels":{"completionCount":"Completion Count","status":"Status","enrollmentCount":"Enrollment Count","courseName":"Course Name","batchName":"Batch Name"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"course_enrollment.csv"}},"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]},"limit":10000}},"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Enrollment Metrics Model","deviceMapping":false}' ;; "course-consumption-report") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseConsumptionJob","modelParams":{"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]}}},"reportConfig":{"id":"tpd_metrics","labels":{"date":"Date","status":"Batch Status","timespent":"Timespent in mins","courseName":"Course Name","batchName":"Batch Name"},"dateRange":{"staticInterval":"LastDay","granularity":"all"},"metrics":[{"metric":"totalCoursePlays","label":"Total Course Plays (in mins)","druidQuery":{"queryType":"groupBy","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"edata_time_spent"}],"dimensions":[{"fieldName":"object_rollup_l1","aliasName":"courseId"},{"fieldName":"uid","aliasName":"userId"},{"fieldName":"context_cdata_id","aliasName":"batchId"}],"filters":[{"type":"equals","dimension":"eid","value":"ME_WORKFLOW_SUMMARY"},{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"context_cdata_type","value":"batch"}],"postAggregation":[{"type":"arithmetic","name":"timespent","fields":{"leftField":"sum__edata_time_spent","rightField":60,"rightFieldType":"constant"},"fn":"/"}],"descending":"false"}}],"output":[{"type":"csv","metrics":["timespent"],"dims":[]}],"queryType":"groupBy"},"store":"{{ dp_object_store_type }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Consumption Metrics Model","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseConsumptionJob","modelParams":{"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]}}},"reportConfig":{"id":"tpd_metrics","labels":{"date":"Date","status":"Batch Status","timespent":"Timespent in mins","courseName":"Course Name","batchName":"Batch Name"},"dateRange":{"staticInterval":"LastDay","granularity":"all"},"metrics":[{"metric":"totalCoursePlays","label":"Total Course Plays (in mins)","druidQuery":{"queryType":"groupBy","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"edata_time_spent"}],"dimensions":[{"fieldName":"object_rollup_l1","aliasName":"courseId"},{"fieldName":"uid","aliasName":"userId"},{"fieldName":"context_cdata_id","aliasName":"batchId"}],"filters":[{"type":"equals","dimension":"eid","value":"ME_WORKFLOW_SUMMARY"},{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"context_cdata_type","value":"batch"}],"postAggregation":[{"type":"arithmetic","name":"timespent","fields":{"leftField":"sum__edata_time_spent","rightField":60,"rightFieldType":"constant"},"fn":"/"}],"descending":"false"}}],"output":[{"type":"csv","metrics":["timespent"],"dims":[]}],"queryType":"groupBy"},"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Consumption Metrics Model","deviceMapping":false}' ;; "textbook-progress-report") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.TextBookProgressModel","modelParams":{"reportConfig":{"id":"content_progress_metrics","metrics":[],"labels":{"board":"Board","medium":"Medium","gradeLevel":"Grade","subject":"Subject","resourceType":"Content Type","totalContent":"Total Contents","live":"Live","review":"Review","draft":"Draft","unlisted":"Limited Sharing","application_ecml":"Created on Diksha","video_youtube":"YouTube Content","video_mp4":"Uploaded Videos","application_pdf":"Text Content","application_html":"Uploaded Interactive Content","identifier":"Content ID","creator":"Created By","createdOn":"Creation Date","lastPublishDate":"Last Publish Date","status":"Status","pkgVersion":"Number of times Published","lastPublishedOn":"Pending in current status since","pendingInCurrentStatus":"Pending in current status since"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"content_progress_metrics.csv","postContainer":"'$reportPostContainer'"}},"filter":{"tenantId":"","slugName":""},"store":"{{ dp_object_store_type }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Progress Metrics Model","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.TextBookProgressModel","modelParams":{"reportConfig":{"id":"content_progress_metrics","metrics":[],"labels":{"board":"Board","medium":"Medium","gradeLevel":"Grade","subject":"Subject","resourceType":"Content Type","totalContent":"Total Contents","live":"Live","review":"Review","draft":"Draft","unlisted":"Limited Sharing","application_ecml":"Created on Diksha","video_youtube":"YouTube Content","video_mp4":"Uploaded Videos","application_pdf":"Text Content","application_html":"Uploaded Interactive Content","identifier":"Content ID","creator":"Created By","createdOn":"Creation Date","lastPublishDate":"Last Publish Date","status":"Status","pkgVersion":"Number of times Published","lastPublishedOn":"Pending in current status since","pendingInCurrentStatus":"Pending in current status since"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"content_progress_metrics.csv","postContainer":"'$reportPostContainer'"}},"filter":{"tenantId":"","slugName":""},"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Progress Metrics Model","deviceMapping":false}' ;; "audit-metrics-report") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.MetricsAuditJob","modelParams":{"auditConfig":[{"name":"denorm","search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"telemetry-denormalized/raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]},"filters":[{"name":"flags.user_data_retrieved","operator":"EQ","value":true},{"name":"flags.content_data_retrieved","operator":"EQ","value":true},{"name":"flags.device_data_retrieved","operator":"EQ","value":true},{"name":"flags.dialcode_data_retrieved","operator":"EQ","value":true},{"name":"flags.collection_data_retrieved","operator":"EQ","value":true},{"name":"flags.derived_location_retrieved","operator":"EQ","value":true}]},{"name":"failed","search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"failed/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"unique","search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"unique/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"raw","search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"channel-raw","search":{"type":"{{ dp_object_store_type }}","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/raw/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"channel-summary","search":{"type":"{{ dp_object_store_type }}","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/summary/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"derived","search":{"type":"{{ dp_object_store_type }}","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"telemetry-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"telemetry-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}},{"name":"summary-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}}]},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$metricsTopic'"}}],"parallelization":8,"appName":"Metrics Audit"}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.MetricsAuditJob","modelParams":{"auditConfig":[{"name":"denorm","search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"telemetry-denormalized/raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]},"filters":[{"name":"flags.user_data_retrieved","operator":"EQ","value":true},{"name":"flags.content_data_retrieved","operator":"EQ","value":true},{"name":"flags.device_data_retrieved","operator":"EQ","value":true},{"name":"flags.dialcode_data_retrieved","operator":"EQ","value":true},{"name":"flags.collection_data_retrieved","operator":"EQ","value":true},{"name":"flags.derived_location_retrieved","operator":"EQ","value":true}]},{"name":"failed","search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"failed/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"unique","search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"unique/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"raw","search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"channel-raw","search":{"type":"{{ cloud_service_provider }}","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/raw/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"channel-summary","search":{"type":"{{ cloud_service_provider }}","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/summary/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"derived","search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"telemetry-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"telemetry-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}},{"name":"summary-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}}]},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$metricsTopic'"}}],"parallelization":8,"appName":"Metrics Audit"}' ;; "sourcing-metrics") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.sourcing.SourcingMetrics","modelParams":{"reportConfig":{"id":"textbook_report","metrics":[],"labels":{"date":"Date","primaryCategory":"Collection Category","identifier":"Collection ID","name":"Collection Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","reportDate":"Report generation date","board":"Board","grade":"Grade","chapters":"Folder Name","totalChapters":"Total number of first level folders","status":"Textbook Status"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]},{"type":"json","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}]},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"createdFor","aliasName":"createdFor"},{"fieldName":"createdOn","aliasName":"createdOn"},{"fieldName":"lastUpdatedOn","aliasName":"lastUpdatedOn"},{"fieldName":"board","aliasName":"board"},{"fieldName":"medium","aliasName":"medium"},{"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName":"gradeLevel","aliasName":"gradeLevel"},{"fieldName":"subject","aliasName":"subject"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"in","dimension":"primaryCategory","values":["Digital Textbook","Course","Content Playlist","Question paper","Question Paper"]},{"type":"in","dimension":"status","values":["Live"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"store":"{{ dp_object_store_type }}","storageContainer":"'$reportPostContainer'","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$reportPostContainer'","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Report Job","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.sourcing.SourcingMetrics","modelParams":{"reportConfig":{"id":"textbook_report","metrics":[],"labels":{"date":"Date","primaryCategory":"Collection Category","identifier":"Collection ID","name":"Collection Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","reportDate":"Report generation date","board":"Board","grade":"Grade","chapters":"Folder Name","totalChapters":"Total number of first level folders","status":"Textbook Status"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]},{"type":"json","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}]},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"createdFor","aliasName":"createdFor"},{"fieldName":"createdOn","aliasName":"createdOn"},{"fieldName":"lastUpdatedOn","aliasName":"lastUpdatedOn"},{"fieldName":"board","aliasName":"board"},{"fieldName":"medium","aliasName":"medium"},{"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName":"gradeLevel","aliasName":"gradeLevel"},{"fieldName":"subject","aliasName":"subject"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"in","dimension":"primaryCategory","values":["Digital Textbook","Course","Content Playlist","Question paper","Question Paper"]},{"type":"in","dimension":"status","values":["Live"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"store":"{{ cloud_service_provider }}","storageContainer":"'$reportPostContainer'","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$reportPostContainer'","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Report Job","deviceMapping":false}' ;; "druid-dataset") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.OnDemandDruidExhaustJob","modelParams":{"store":"{{ dp_object_store_type }}","container":"'$reportPostContainer'","key":"ml_reports/","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ML Druid Data Model"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.OnDemandDruidExhaustJob","modelParams":{"store":"{{ cloud_service_provider }}","container":"'$reportPostContainer'","key":"ml_reports/","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ML Druid Data Model"}' ;; "*") echo "Unknown model code" diff --git a/ansible/roles/data-products-deploy/templates/model-config.json.j2 b/ansible/roles/data-products-deploy/templates/model-config.json.j2 index bb170b2193..895b41186d 100644 --- a/ansible/roles/data-products-deploy/templates/model-config.json.j2 +++ b/ansible/roles/data-products-deploy/templates/model-config.json.j2 @@ -1,7 +1,7 @@ { "wfs": { "search": { - "type": "{{dp_object_store_type}}", + "type": "{{cloud_service_provider}}", "queries": [ { "bucket": "{{ bucket }}", @@ -25,11 +25,11 @@ "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", "storageEndpoint":"{{dp_storage_endpoint_config}}", - "store":"{{ dp_object_store_type }}" + "store":"{{ cloud_service_provider }}" }, "output": [ { - "to": "{{dp_object_store_type}}", + "to": "{{cloud_service_provider}}", "params": { "bucket": "{{ bucket }}", "key": "{{ job_manager_tmp_dir }}/wfs/$(date --date yesterday '+%Y-%m-%d')" @@ -49,7 +49,7 @@ }, "video-streaming": { "search": { - "type": "{{dp_object_store_type}}" + "type": "{{cloud_service_provider}}" }, "model": "org.ekstep.analytics.job.VideoStreamingJob", "modelParams": { @@ -77,7 +77,7 @@ "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", "storageEndpoint":"{{dp_storage_endpoint_config}}", - "store":"{{ dp_object_store_type }}", + "store":"{{ cloud_service_provider }}", "sparkCassandraConnectionHost": "{{core_cassandra_host}}", "sparkElasticsearchConnectionHost": "{{sunbird_es_host}}" }, @@ -103,7 +103,7 @@ "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", "storageEndpoint":"{{dp_storage_endpoint_config}}", - "store":"{{ dp_object_store_type }}", + "store":"{{ cloud_service_provider }}", "sparkCassandraConnectionHost": "{{core_cassandra_host}}", "sparkElasticsearchConnectionHost": "{{sunbird_es_host}}" }, @@ -125,7 +125,7 @@ }, "model": "org.ekstep.analytics.updater.UpdateContentRating", "modelParams": { - "store": "{{dp_object_store_type}}", + "store": "{{cloud_service_provider}}", "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", @@ -163,7 +163,7 @@ "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", "storageEndpoint":"{{dp_storage_endpoint_config}}", - "store":"{{ dp_object_store_type }}", + "store":"{{ cloud_service_provider }}", "model": [ { "model": "WorkFlowSummaryModel", @@ -226,7 +226,7 @@ }, "model": "org.ekstep.analytics.model.ExperimentDefinitionModel", "modelParams": { - "store": "{{dp_object_store_type}}", + "store": "{{cloud_service_provider}}", "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", @@ -327,7 +327,7 @@ "tenantId": "", "slugName": "" }, - "store": "{{dp_object_store_type}}", + "store": "{{cloud_service_provider}}", "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", @@ -380,7 +380,7 @@ "limit": 10000 } }, - "store": "{{dp_object_store_type}}", + "store": "{{cloud_service_provider}}", "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", @@ -495,7 +495,7 @@ }], "queryType": "groupBy" }, - "store": "{{dp_object_store_type}}", + "store": "{{cloud_service_provider}}", "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", @@ -524,7 +524,7 @@ }, "model": "org.ekstep.analytics.model.MetricsAuditJob", "modelParams": { - "store": "{{dp_object_store_type}}", + "store": "{{cloud_service_provider}}", "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", @@ -533,7 +533,7 @@ { "name": "denorm", "search": { - "type": "{{dp_object_store_type}}", + "type": "{{cloud_service_provider}}", "queries": [ { "bucket": "{{ bucket }}", @@ -579,7 +579,7 @@ { "name": "failed", "search": { - "type": "{{dp_object_store_type}}", + "type": "{{cloud_service_provider}}", "queries": [ { "bucket": "{{ bucket }}", @@ -593,7 +593,7 @@ { "name": "unique", "search": { - "type": "{{dp_object_store_type}}", + "type": "{{cloud_service_provider}}", "queries": [ { "bucket": "{{ bucket }}", @@ -607,7 +607,7 @@ { "name": "raw", "search": { - "type": "{{dp_object_store_type}}", + "type": "{{cloud_service_provider}}", "queries": [ { "bucket": "{{ bucket }}", @@ -621,7 +621,7 @@ { "name": "channel-raw", "search": { - "type": "{{dp_object_store_type}}", + "type": "{{cloud_service_provider}}", "queries": [ { "folder": true, @@ -636,7 +636,7 @@ { "name": "channel-summary", "search": { - "type": "{{dp_object_store_type}}", + "type": "{{cloud_service_provider}}", "queries": [ { "folder": true, @@ -651,7 +651,7 @@ { "name": "derived", "search": { - "type": "{{dp_object_store_type}}", + "type": "{{cloud_service_provider}}", "queries": [ { "bucket": "{{ bucket }}", diff --git a/ansible/roles/data-products-deploy/templates/model-dock-config.j2 b/ansible/roles/data-products-deploy/templates/model-dock-config.j2 index f720f4687e..af8ada93ee 100644 --- a/ansible/roles/data-products-deploy/templates/model-dock-config.j2 +++ b/ansible/roles/data-products-deploy/templates/model-dock-config.j2 @@ -15,16 +15,16 @@ config() { if [ ! -z "$3" ]; then inputBucket=$3; fi case "$1" in "content-details") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.sourcing.ContentDetailsReport","modelParams":{"tenantId":"","slug":"","reportConfig":{"id":"content_report","metrics":[],"labels":{"programName":"Project Name","programId":"Project ID","contentId":"Content/Question ID","contentName":"Content/Question Name","mimeType":"MimeType","chapterId":"Folder ID","contentStatus":"Content/Question Status","creator":"Creator Name","createdBy":"CreatedBy ID","date":"Date","identifier":"Collection/Question Set ID","name":"Collection/Question Set Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","board":"Board","grade":"Grade","chapters":"Chapter Name","status":"Textbook Status","objectType":"Object Type","primaryCategory":"Primary category","topic":"Topic","learningOutcome":"Learning Outcome","addedFromLibrary":"Added from library","contentType":"Content Type"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]},{"type":"json","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}]},"contentQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"unitIdentifiers","aliasName":"unitIdentifiers"},{"fieldName":"collectionId","aliasName":"collectionId"},{"fieldName":"createdBy","aliasName":"createdBy"},{"fieldName":"creator","aliasName":"creator"},{"fieldName":"mimeType","aliasName":"mimeType"},{"fieldName":"topic","aliasName":"topic"},{"fieldName":"learningOutcome","aliasName":"learningOutcome"},{"fieldName":"primaryCategory","aliasName":"contentType"}],"filters":[{"type":"notequals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live"]},{"type":"isnotnull","dimension":"collectionId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"textbookQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"programId","aliasName":"programId"},{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"board","aliasName":"board"},{"fieldName":"medium","aliasName":"medium"},{"fieldName":"gradeLevel","aliasName":"gradeLevel"},{"fieldName":"subject","aliasName":"subject"},{"fieldName":"status","aliasName":"status"},{"fieldName":"acceptedContents","aliasName":"acceptedContents"},{"fieldName":"acceptedContributions","aliasName":"acceptedContributions"},{"fieldName":"rejectedContents","aliasName":"rejectedContents"},{"fieldName":"rejectedContributions","aliasName":"rejectedContributions"},{"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName":"objectType","aliasName":"objectType"},{"fieldName":"reusedContributions","aliasName":"reusedContributions"}],"filters":[{"type":"in","dimension":"primaryCategory","values":["Digital Textbook","Course","Content Playlist","Question paper","Question Paper","Exam Question Set","Practice Set","Demo Practice Question Set"]},{"type":"isnotnull","dimension":"programId"},{"type":"in","dimension":"status","values":["Draft"]},{"type":"equals","dimension":"channel","value":"channelId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"store":"{{dp_object_store_type}}","storageKeyConfig":"azure_storage_key","storageSecretConfig":"azure_storage_secret","storageContainer":"'$reportPostContainer'","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$reportPostContainer'","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Content Report Job","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.sourcing.ContentDetailsReport","modelParams":{"tenantId":"","slug":"","reportConfig":{"id":"content_report","metrics":[],"labels":{"programName":"Project Name","programId":"Project ID","contentId":"Content/Question ID","contentName":"Content/Question Name","mimeType":"MimeType","chapterId":"Folder ID","contentStatus":"Content/Question Status","creator":"Creator Name","createdBy":"CreatedBy ID","date":"Date","identifier":"Collection/Question Set ID","name":"Collection/Question Set Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","board":"Board","grade":"Grade","chapters":"Chapter Name","status":"Textbook Status","objectType":"Object Type","primaryCategory":"Primary category","topic":"Topic","learningOutcome":"Learning Outcome","addedFromLibrary":"Added from library","contentType":"Content Type"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]},{"type":"json","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}]},"contentQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"unitIdentifiers","aliasName":"unitIdentifiers"},{"fieldName":"collectionId","aliasName":"collectionId"},{"fieldName":"createdBy","aliasName":"createdBy"},{"fieldName":"creator","aliasName":"creator"},{"fieldName":"mimeType","aliasName":"mimeType"},{"fieldName":"topic","aliasName":"topic"},{"fieldName":"learningOutcome","aliasName":"learningOutcome"},{"fieldName":"primaryCategory","aliasName":"contentType"}],"filters":[{"type":"notequals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live"]},{"type":"isnotnull","dimension":"collectionId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"textbookQuery":{"queryType":"groupBy","dataSource":"vdn-content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"programId","aliasName":"programId"},{"fieldName":"identifier","aliasName":"identifier"},{"fieldName":"name","aliasName":"name"},{"fieldName":"board","aliasName":"board"},{"fieldName":"medium","aliasName":"medium"},{"fieldName":"gradeLevel","aliasName":"gradeLevel"},{"fieldName":"subject","aliasName":"subject"},{"fieldName":"status","aliasName":"status"},{"fieldName":"acceptedContents","aliasName":"acceptedContents"},{"fieldName":"acceptedContributions","aliasName":"acceptedContributions"},{"fieldName":"rejectedContents","aliasName":"rejectedContents"},{"fieldName":"rejectedContributions","aliasName":"rejectedContributions"},{"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName":"objectType","aliasName":"objectType"},{"fieldName":"reusedContributions","aliasName":"reusedContributions"}],"filters":[{"type":"in","dimension":"primaryCategory","values":["Digital Textbook","Course","Content Playlist","Question paper","Question Paper","Exam Question Set","Practice Set","Demo Practice Question Set"]},{"type":"isnotnull","dimension":"programId"},{"type":"in","dimension":"status","values":["Draft"]},{"type":"equals","dimension":"channel","value":"channelId"}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"store":"{{cloud_service_provider}}","storageKeyConfig":"azure_storage_key","storageSecretConfig":"azure_storage_secret","storageContainer":"'$reportPostContainer'","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$reportPostContainer'","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Content Report Job","deviceMapping":false}' ;; "sourcing-summary-report") - echo '{"search": {"type": "none"}, "model": "org.ekstep.analytics.job.report.SourcingSummaryReport", "modelParams": {"storageKeyConfig":"druid_storage_account_key", "storageSecretConfig":"druid_storage_account_secret", "dataSource": "sourcing-summary-snapshot", "druidHost": "'$druidRollupHost'", "druidSegmentUrl":"'$druidRollupHost'/druid/coordinator/v1/metadata/datasources/sourcing-model-snapshot/segments", "deleteSegmentUrl": "'$druidRollupHost'/druid/coordinator/v1/datasources/sourcing-model-snapshot/segments/", "druidIngestionUrl": "'$druidRollupHost'/druid/indexer/v1/task", "specPath": "/mount/data/analytics/scripts/sourcing-ingestion-spec.json", "dbName": "opensaberdb", "tables": {"programTable": "program", "nominationTable": "nomination"}, "druidQuery": {"queryType": "groupBy", "dataSource": "vdn-content-model-snapshot", "intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00", "aggregations": [{"name": "count", "type": "count"}], "dimensions": [{"fieldName": "primaryCategory", "aliasName": "primaryCategory"}, {"fieldName": "createdBy", "aliasName": "createdBy"}], "filters": [{"type": "equals", "dimension": "objectType", "value": "Content"}, {"type": "equals", "dimension": "sampleContent", "value": "false"}], "postAggregation": [], "descending": "false", "limitSpec": {"type": "default", "limit": 1000000, "columns": [{"dimension": "count", "direction": "descending"}]}}, "reportConfig": {"id": "sourcing", "metrics": [], "labels": {}, "output": [{"type": "json", "dims": ["identifier", "channel", "name"], "fileParameters": ["id", "dims"]}]}, "store": "{{dp_object_store_type}}", "format": "json", "folderPrefix": ["slug", "reportName"]}, "output": [{"to": "console", "params": {"printEvent": false}}], "parallelization": 8, "appName": "Sourcing Summary Report Job", "deviceMapping": false}' + echo '{"search": {"type": "none"}, "model": "org.ekstep.analytics.job.report.SourcingSummaryReport", "modelParams": {"storageKeyConfig":"druid_storage_account_key", "storageSecretConfig":"druid_storage_account_secret", "dataSource": "sourcing-summary-snapshot", "druidHost": "'$druidRollupHost'", "druidSegmentUrl":"'$druidRollupHost'/druid/coordinator/v1/metadata/datasources/sourcing-model-snapshot/segments", "deleteSegmentUrl": "'$druidRollupHost'/druid/coordinator/v1/datasources/sourcing-model-snapshot/segments/", "druidIngestionUrl": "'$druidRollupHost'/druid/indexer/v1/task", "specPath": "/mount/data/analytics/scripts/sourcing-ingestion-spec.json", "dbName": "opensaberdb", "tables": {"programTable": "program", "nominationTable": "nomination"}, "druidQuery": {"queryType": "groupBy", "dataSource": "vdn-content-model-snapshot", "intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00", "aggregations": [{"name": "count", "type": "count"}], "dimensions": [{"fieldName": "primaryCategory", "aliasName": "primaryCategory"}, {"fieldName": "createdBy", "aliasName": "createdBy"}], "filters": [{"type": "equals", "dimension": "objectType", "value": "Content"}, {"type": "equals", "dimension": "sampleContent", "value": "false"}], "postAggregation": [], "descending": "false", "limitSpec": {"type": "default", "limit": 1000000, "columns": [{"dimension": "count", "direction": "descending"}]}}, "reportConfig": {"id": "sourcing", "metrics": [], "labels": {}, "output": [{"type": "json", "dims": ["identifier", "channel", "name"], "fileParameters": ["id", "dims"]}]}, "store": "{{cloud_service_provider}}", "format": "json", "folderPrefix": ["slug", "reportName"]}, "output": [{"to": "console", "params": {"printEvent": false}}], "parallelization": 8, "appName": "Sourcing Summary Report Job", "deviceMapping": false}' ;; "funnel-report") - echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.job.report.FunnelReport","modelParams": {"contributionConfig": {"contentRequest": {"request": {"filters": {"programId": "programIdentifier","objectType": "content","status": ["Draft", "Live", "Review"],"mimeType": "application/vnd.ekstep.content-collection"},"fields": ["acceptedContents", "rejectedContents"],"limit": 10000}},"correctionsPendingRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": "Draft","prevStatus": "Live","programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}},"contributionRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": ["Live"],"programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}}},"reportConfig": {"id": "funnel_report","metrics": [],"labels": {"reportDate": "Report generation date","visitors": "No. of users opening the project","projectName": "Project Name","initiatedNominations": "No. of initiated nominations","rejectedNominations": "No. of rejected nominations","pendingNominations": "No. of nominations pending review","acceptedNominations": "No. of accepted nominations to the project","noOfContributors": "No. of contributors to the project","noOfContributions": "No. of contributions to the project","pendingContributions": "No. of contributions pending review","approvedContributions": "No. of approved contributions"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"store": "{{dp_object_store_type}}","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","folderPrefix": ["slug", "reportName"]},"sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","druidConfig": {"queryType": "timeseries","dataSource": "telemetry-events-syncts","intervals": "startdate/enddate","aggregations": [{"name": "visitors","type": "count","fieldName": "actor_id"}],"filters": [{"type": "equals","dimension": "context_cdata_id","value": "program_id"}, {"type": "equals","dimension": "edata_pageid","value": "contribution_project_contributions"}, {"type": "equals","dimension": "context_pdata_pid","value": "creation-portal.programs"}, {"type": "equals","dimension": "context_cdata_type","value": "project"}, {"type": "equals","dimension": "context_env","value": "creation-portal"}, {"type": "equals","dimension": "eid","value": "IMPRESSION"}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Funnel Report Job","deviceMapping": false}' + echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.job.report.FunnelReport","modelParams": {"contributionConfig": {"contentRequest": {"request": {"filters": {"programId": "programIdentifier","objectType": "content","status": ["Draft", "Live", "Review"],"mimeType": "application/vnd.ekstep.content-collection"},"fields": ["acceptedContents", "rejectedContents"],"limit": 10000}},"correctionsPendingRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": "Draft","prevStatus": "Live","programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}},"contributionRequest": {"request": {"filters": {"objectType": ["content","questionset"],"status": ["Live"],"programId": "programIdentifier","mimeType": {"!=": "application/vnd.ekstep.content-collection"},"contentType": {"!=": "Asset"}},"not_exists": ["sampleContent"],"facets": ["createdBy"],"limit": 0}}},"reportConfig": {"id": "funnel_report","metrics": [],"labels": {"reportDate": "Report generation date","visitors": "No. of users opening the project","projectName": "Project Name","initiatedNominations": "No. of initiated nominations","rejectedNominations": "No. of rejected nominations","pendingNominations": "No. of nominations pending review","acceptedNominations": "No. of accepted nominations to the project","noOfContributors": "No. of contributors to the project","noOfContributions": "No. of contributions to the project","pendingContributions": "No. of contributions pending review","approvedContributions": "No. of approved contributions"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"store": "{{cloud_service_provider}}","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","folderPrefix": ["slug", "reportName"]},"sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","druidConfig": {"queryType": "timeseries","dataSource": "telemetry-events-syncts","intervals": "startdate/enddate","aggregations": [{"name": "visitors","type": "count","fieldName": "actor_id"}],"filters": [{"type": "equals","dimension": "context_cdata_id","value": "program_id"}, {"type": "equals","dimension": "edata_pageid","value": "contribution_project_contributions"}, {"type": "equals","dimension": "context_pdata_pid","value": "creation-portal.programs"}, {"type": "equals","dimension": "context_cdata_type","value": "project"}, {"type": "equals","dimension": "context_env","value": "creation-portal"}, {"type": "equals","dimension": "eid","value": "IMPRESSION"}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Funnel Report Job","deviceMapping": false}' ;; "sourcing-metrics") - echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.sourcing.SourcingMetrics","modelParams": {"reportConfig": {"id": "textbook_report","metrics": [],"labels": {"date": "Date","identifier": "Textbook ID","name": "Textbook Name","medium": "Medium","gradeLevel": "Grade","subject": "Subject","createdOn": "Created On","lastUpdatedOn": "Last Updated On","reportDate": "Report generation date","board": "Board","grade": "Grade","chapters": "Chapter Name","totalChapters": "Total number of chapters (first level sections of ToC)","status": "Textbook Status"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"druidConfig": {"queryType": "groupBy","dataSource": "content-model-snapshot","intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations": [{"name": "count","type": "count"}],"dimensions": [{"fieldName": "channel","aliasName": "channel"}, {"fieldName": "identifier","aliasName": "identifier","type": "Extraction","outputType": "STRING","extractionFn": [{"type": "javascript","fn": "function(str){return str == null ? null: str.split(\".\")[0]}"}]}, {"fieldName": "name","aliasName": "name"}, {"fieldName": "createdFor","aliasName": "createdFor"}, {"fieldName": "createdOn","aliasName": "createdOn"}, {"fieldName": "lastUpdatedOn","aliasName": "lastUpdatedOn"}, {"fieldName": "board","aliasName": "board"}, {"fieldName": "medium","aliasName": "medium"}, {"fieldName": "gradeLevel","aliasName": "gradeLevel"}, {"fieldName": "subject","aliasName": "subject"}, {"fieldName": "status","aliasName": "status"}],"filters": [{"type": "equals","dimension": "contentType","value": "TextBook"}, {"type": "in","dimension": "status","values": ["Live"]}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"store": "{{dp_object_store_type}}","storageContainer": "'$reportPostContainer'","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","folderPrefix": ["slug", "reportName"]},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Textbook Report Job","deviceMapping": false}' + echo '{"search": {"type": "none"},"model": "org.ekstep.analytics.sourcing.SourcingMetrics","modelParams": {"reportConfig": {"id": "textbook_report","metrics": [],"labels": {"date": "Date","identifier": "Textbook ID","name": "Textbook Name","medium": "Medium","gradeLevel": "Grade","subject": "Subject","createdOn": "Created On","lastUpdatedOn": "Last Updated On","reportDate": "Report generation date","board": "Board","grade": "Grade","chapters": "Chapter Name","totalChapters": "Total number of chapters (first level sections of ToC)","status": "Textbook Status"},"output": [{"type": "csv","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}, {"type": "json","dims": ["identifier", "channel", "name"],"fileParameters": ["id", "dims"]}]},"druidConfig": {"queryType": "groupBy","dataSource": "content-model-snapshot","intervals": "1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations": [{"name": "count","type": "count"}],"dimensions": [{"fieldName": "channel","aliasName": "channel"}, {"fieldName": "identifier","aliasName": "identifier","type": "Extraction","outputType": "STRING","extractionFn": [{"type": "javascript","fn": "function(str){return str == null ? null: str.split(\".\")[0]}"}]}, {"fieldName": "name","aliasName": "name"}, {"fieldName": "createdFor","aliasName": "createdFor"}, {"fieldName": "createdOn","aliasName": "createdOn"}, {"fieldName": "lastUpdatedOn","aliasName": "lastUpdatedOn"}, {"fieldName": "board","aliasName": "board"}, {"fieldName": "medium","aliasName": "medium"}, {"fieldName": "gradeLevel","aliasName": "gradeLevel"}, {"fieldName": "subject","aliasName": "subject"}, {"fieldName": "status","aliasName": "status"}],"filters": [{"type": "equals","dimension": "contentType","value": "TextBook"}, {"type": "in","dimension": "status","values": ["Live"]}],"postAggregation": [],"descending": "false","limitSpec": {"type": "default","limit": 1000000,"columns": [{"dimension": "count","direction": "descending"}]}},"store": "{{cloud_service_provider}}","storageContainer": "'$reportPostContainer'","format": "csv","key": "druid-reports/","filePath": "druid-reports/","container": "'$reportPostContainer'","sparkCassandraConnectionHost": "'$sunbirdPlatformCassandraHost'","folderPrefix": ["slug", "reportName"]},"output": [{"to": "console","params": {"printEvent": false}}],"parallelization": 8,"appName": "Textbook Report Job","deviceMapping": false}' ;; "*") echo "Unknown model code" diff --git a/ansible/roles/data-products-deploy/templates/sourcing-ingestion-spec.j2 b/ansible/roles/data-products-deploy/templates/sourcing-ingestion-spec.j2 index 41bb51afba..2afee4d467 100644 --- a/ansible/roles/data-products-deploy/templates/sourcing-ingestion-spec.j2 +++ b/ansible/roles/data-products-deploy/templates/sourcing-ingestion-spec.j2 @@ -125,7 +125,7 @@ "ioConfig": { "type": "index", "firehose": { -{% if dp_object_store_type == "azure" %} +{% if cloud_service_provider == "azure" %} "type": "static-azure-blobstore", "blobs": [ { @@ -133,7 +133,7 @@ "path": "/sourcing/SourcingSummaryReport.json" } ], -{% elif (dp_object_store_type == "oci") %} +{% elif (cloud_service_provider == "oci") %} "type": "static-s3", "uris": [ "s3://{{reports_container}}/sourcing/SourcingSummaryReport.json"], {% endif %} diff --git a/ansible/roles/data-products-deploy/templates/submit-script.j2 b/ansible/roles/data-products-deploy/templates/submit-script.j2 index 0e629a4ce8..6a65482bd6 100644 --- a/ansible/roles/data-products-deploy/templates/submit-script.j2 +++ b/ansible/roles/data-products-deploy/templates/submit-script.j2 @@ -80,11 +80,11 @@ submit_cluster_job() { requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr} finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable} echo $finalRequestBody -{% if dp_object_store_type == "azure" %} +{% if cloud_service_provider == "azure" %} { curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}" } -{% elif (dp_object_store_type == "oci") %} +{% elif (cloud_service_provider == "oci") %} { curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_bds_url }}:8998/batches' -H "X-Requested-By: {{ admin_name }}" } @@ -127,11 +127,11 @@ if [ "$mode" = "via-partition" ]; then requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr} finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable} echo $finalRequestBody -{% if dp_object_store_type == "azure" %} +{% if cloud_service_provider == "azure" %} { curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}" } -{% elif (dp_object_store_type == "oci") %} +{% elif (cloud_service_provider == "oci") %} { curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_bds_url }}:8998/batches' -H "X-Requested-By: {{ admin_name }}" } @@ -174,11 +174,11 @@ elif [ "$mode" = "selected-partition" ]; then requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr} finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable} echo $finalRequestBody -{% if dp_object_store_type == "azure" %} +{% if cloud_service_provider == "azure" %} { curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}" } -{% elif (dp_object_store_type == "oci") %} +{% elif (cloud_service_provider == "oci") %} { curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_bds_url }}:8998/batches' -H "X-Requested-By: {{ admin_name }}" } @@ -203,11 +203,11 @@ else requestBody=${clusterConfig/'"className": "org.ekstep.analytics.job.JobExecutor"'/$argsStr} finalRequestBody=${requestBody/'org.ekstep.analytics.job.JobExecutor'/$classVariable} echo $finalRequestBody -{% if dp_object_store_type == "azure" %} +{% if cloud_service_provider == "azure" %} { curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_cluster_name }}.azurehdinsight.net/livy/batches' -H "X-Requested-By: {{ admin_name }}" } -{% elif (dp_object_store_type == "oci") %} +{% elif (cloud_service_provider == "oci") %} { curl -k --user "{{ admin_name }}:{{ admin_password }}" -v -H "Content-Type: application/json" -X POST -d "$finalRequestBody" 'https://{{ spark_bds_url }}:8998/batches' -H "X-Requested-By: {{ admin_name }}" } From 33c89a36cb308c0313cb2c449d65181c58aba83b Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 21 Nov 2023 14:46:06 +0530 Subject: [PATCH 11/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/roles/data-products-deploy/templates/common.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/data-products-deploy/templates/common.conf.j2 b/ansible/roles/data-products-deploy/templates/common.conf.j2 index ae013008aa..5be5921e77 100644 --- a/ansible/roles/data-products-deploy/templates/common.conf.j2 +++ b/ansible/roles/data-products-deploy/templates/common.conf.j2 @@ -216,7 +216,7 @@ metric.kafka.broker="{{groups['processing-cluster-kafka']|join(':9092,')}}:9092" metric.kafka.topic="{{ env }}.prom.monitoring.metrics" //Postgres Config -postgres.db="{{postgres.spark_db_name}}" +postgres.db="{{postgres.db_name}}" postgres.url="jdbc:postgresql://{{postgres.db_url}}:{{postgres.db_port}}/" postgres.user="{{postgres.db_username}}" postgres.pass="{{postgres.db_password}}" From 23cb2f8937b3c42fa2ec932ecd28811fe6f05b92 Mon Sep 17 00:00:00 2001 From: SadanandGowda Date: Tue, 21 Nov 2023 20:14:22 +0530 Subject: [PATCH 12/46] added cloud_storage_privatereports_bucketname variable --- ansible/roles/data-products-deploy/templates/common.conf.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/roles/data-products-deploy/templates/common.conf.j2 b/ansible/roles/data-products-deploy/templates/common.conf.j2 index 5be5921e77..4484e19641 100644 --- a/ansible/roles/data-products-deploy/templates/common.conf.j2 +++ b/ansible/roles/data-products-deploy/templates/common.conf.j2 @@ -256,7 +256,7 @@ druid.report.default.storage="s3" {% endif %} druid.report.date.format="yyyy-MM-dd" -druid.report.default.container="{{cloud_storage_report_verfication_bucketname}}" +druid.report.default.container="{{cloud_storage_privatereports_bucketname}}" ## Collection Exhaust Jobs Configuration -- Start ## From b80641276b0abaf55c87800a06bab29da9f8e9bd Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Wed, 22 Nov 2023 13:17:14 +0530 Subject: [PATCH 13/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/roles/data-products-deploy/defaults/main.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/ansible/roles/data-products-deploy/defaults/main.yml b/ansible/roles/data-products-deploy/defaults/main.yml index a564f3c66a..e67474ffd0 100755 --- a/ansible/roles/data-products-deploy/defaults/main.yml +++ b/ansible/roles/data-products-deploy/defaults/main.yml @@ -246,6 +246,7 @@ exhaust_user_parallelism: 200 data_exhaust_batch_limit_per_request: 20 # Start Of UCI Related Variables +uci_env: "{{ env }}" uci_postgres_host: "dev-pg11.postgres.database.azure.com" uci_encryption_key_base64: "" uci_bot_postgres_database: uci-botdb @@ -292,4 +293,5 @@ jets3t_s3_https_only: "{{ s3_https_only }}" jets3t_s3_default_bucket_location: "{{ s3_default_bucket_location }}" -spark_pg_db_name: "analytics" \ No newline at end of file +spark_pg_db_name: "analytics" +spark_bds_url: "" # spark cluster URL. Similar to azure hdInsight cluster/ aws EMR cluster \ No newline at end of file From d78959f85321403c7df5deab8c31bb62af333ef9 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Wed, 22 Nov 2023 13:34:21 +0530 Subject: [PATCH 14/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/roles/data-products-deploy/defaults/main.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/roles/data-products-deploy/defaults/main.yml b/ansible/roles/data-products-deploy/defaults/main.yml index e67474ffd0..9ef231344a 100755 --- a/ansible/roles/data-products-deploy/defaults/main.yml +++ b/ansible/roles/data-products-deploy/defaults/main.yml @@ -247,6 +247,7 @@ data_exhaust_batch_limit_per_request: 20 # Start Of UCI Related Variables uci_env: "{{ env }}" +sunbird_instance: "{{ env }}" uci_postgres_host: "dev-pg11.postgres.database.azure.com" uci_encryption_key_base64: "" uci_bot_postgres_database: uci-botdb From e2208fbb633e101dcad604c00f089d379a45a73d Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Wed, 22 Nov 2023 13:41:18 +0530 Subject: [PATCH 15/46] #OB-553 fix: Deployment issue fixes for OCI --- .../templates/model-config.j2 | 40 +++++++++---------- .../templates/model-config.json.j2 | 20 +++++----- 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/ansible/roles/data-products-deploy/templates/model-config.j2 b/ansible/roles/data-products-deploy/templates/model-config.j2 index 5a7f3d2c4d..6fc260b343 100644 --- a/ansible/roles/data-products-deploy/templates/model-config.j2 +++ b/ansible/roles/data-products-deploy/templates/model-config.j2 @@ -33,7 +33,7 @@ config() { echo '{"search":{"type":"{{cloud_service_provider}}","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"model":"org.sunbird.analytics.model.report.AssessmentCorrectionModel","modelParams":{"parallelization":200,"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","granularity":"all","aggregations":[{"name":"count","type":"count","fieldName":"count"}],"dimensions":[{"fieldName":"identifier","aliasName":"identifier"}],"filters":[{"type":"equals","dimension":"contentType","value":"SelfAssess"}],"descending":"false"},"fileOutputConfig":{"to":"file","params":{"file":"{{ analytics.home }}/assessment-correction/skippedEvents"}},"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"kafka","params":{"brokerList":"'$brokerIngestionList'","topic":"'$assessTopic'"}}],"parallelization":200,"appName":"Assessment Correction Model"}' ;; "assessment-archival") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Job"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.AssessmentArchivalJob","modelParams":{"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Assessment Archival Job"}' ;; "assessment-archived-removal") {% if cloud_service_provider == "azure" %} @@ -47,7 +47,7 @@ config() { echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CollectionReconciliationJob","modelParams":{"mode":"prodrun","brokerList":"{{ingestion_kafka_broker_host}}","topic":"{{env}}.issue.certificate.request","sparkCassandraConnectionHost":"{{ core_cassandra_host }}"},"parallelization":30,"appName":"CollectionReconciliationJob"}' ;; "collection-summary-report") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course"},"fields":["identifier","name","organisation","channel"],"limit":10000}},"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course"},"fields":["identifier","name","organisation","channel"],"limit":10000}},"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' ;; "score-metric-migration-job") echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.ScoreMetricMigrationJob","modelParams":{"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Score Metric Migration Job"}' @@ -56,34 +56,34 @@ config() { echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.AssessmentScoreCorrectionJob","modelParams":{"assessment.score.correction.batches":"","cassandraReadConsistency":"QUORUM","cassandraWriteConsistency":"QUORUM","csvPath":"/mount/data/analytics/score_correction","isDryRunMode":true,"sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":30,"appName":"Assessment Score Correction Job"}' ;; "course-batch-status-updater") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CourseBatchStatusUpdaterJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"http://{{ single_node_es_host }}:9200","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","kpLearningBasePath":"http://{{groups['learning'][0]}}:8080/learning-service","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Course Batch Status Updater Job"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.audit.CourseBatchStatusUpdaterJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","sparkElasticsearchConnectionHost":"http://{{ single_node_es_host }}:9200","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","kpLearningBasePath":"http://{{groups['learning'][0]}}:8080/learning-service","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Course Batch Status Updater Job"}' ;; "collection-summary-report-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJobV2","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","batchSize":50,"generateForAllBatches":true,"contentFields":["identifier","name","organisation","channel","status","keywords","createdFor","medium","subject"],"contentStatus":["Live","Unlisted","Retired"],"store":"{{cloud_service_provider}}","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","specPath":"/mount/data/analytics/scripts/collection-summary-ingestion-spec.json","druidIngestionUrl":"'$druidIngestionURL'","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report V2"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJobV2","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","batchSize":50,"generateForAllBatches":true,"contentFields":["identifier","name","organisation","channel","status","keywords","createdFor","medium","subject"],"contentStatus":["Live","Unlisted","Retired"],"store":"{{cloud_service_provider}}","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","specPath":"/mount/data/analytics/scripts/collection-summary-ingestion-spec.json","druidIngestionUrl":"'$druidIngestionURL'","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report V2"}' ;; "uci-private-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.uci.UCIPrivateExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UCI Private Exhaust"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.uci.UCIPrivateExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","mode":"OnDemand","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UCI Private Exhaust"}' ;; "uci-response-exhaust") - echo '{"search":{"type":"{{cloud_service_provider}}","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"filters":[{"name":"eid","operator":"EQ","value":"ASSESS"}],"model":"org.sunbird.analytics.uci.UCIResponseExhaust","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","botPdataId":"{{ uci_pdata_id }}","mode":"OnDemand","fromDate":"","toDate":""},"parallelization":8,"appName":"UCI Response Exhaust"}' + echo '{"search":{"type":"{{cloud_service_provider}}","queries":[{"bucket":"'$bucket'","prefix":"unique/raw/","endDate":"'$endDate'","delta":0}]},"filters":[{"name":"eid","operator":"EQ","value":"ASSESS"}],"model":"org.sunbird.analytics.uci.UCIResponseExhaust","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","botPdataId":"{{ uci_pdata_id }}","mode":"OnDemand","fromDate":"","toDate":""},"parallelization":8,"appName":"UCI Response Exhaust"}' ;; "userinfo-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UserInfo Exhaust"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.UserInfoExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"UserInfo Exhaust"}' ;; "program-collection-summary-report") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course","keywords":["'$keyword'"]},"fields":["identifier","name","organisation","channel"],"limit":10000}},"columns":["Published by","Batch id","Collection id","Collection name","Batch start date","Batch end date","State","Total enrolments By State","Total completion By State"],"keywords":"'$keyword'","store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.job.report.CollectionSummaryJob","modelParams":{"searchFilter":{"request":{"filters":{"status":["Live"],"contentType":"Course","keywords":["'$keyword'"]},"fields":["identifier","name","organisation","channel"],"limit":10000}},"columns":["Published by","Batch id","Collection id","Collection name","Batch start date","Batch end date","State","Total enrolments By State","Total completion By State"],"keywords":"'$keyword'","store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Collection Summary Report"}' ;; "response-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust"}' ;; "response-exhaust-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust V2"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ResponseExhaustJobV2","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Response Exhaust V2"}' ;; "progress-exhaust") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}", "apiVersion":"v2", "parallelization":200,"mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}", "apiVersion":"v2", "parallelization":200,"mode":"OnDemand","batchFilters":["TPD"],"searchFilter":{}, "sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}", "sparkCassandraConnectionHost":"{{ core_cassandra_host }}", "fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust"}' ;; "progress-exhaust-v2") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2","modelParams":{"store":"{{cloud_service_provider}}","mode":"OnDemand","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust V2"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.collection.ProgressExhaustJobV2","modelParams":{"store":"{{cloud_service_provider}}","mode":"OnDemand","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","batchFilters":["TPD"],"searchFilter":{},"sparkElasticsearchConnectionHost":"{{ sunbird_es_host }}","sparkRedisConnectionHost":"{{ metadata2_redis_host }}","sparkUserDbRedisIndex":"12","sparkUserDbRedisPort":"{{ user_port }}","sparkCassandraConnectionHost":"{{ core_cassandra_host }}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')"},"parallelization":8,"appName":"Progress Exhaust V2"}' ;; "druid_reports") echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.DruidQueryProcessingModel","modelParams":{"mode":"batch"},"parallelization":8,"appName":"Druid Reports"}' @@ -98,17 +98,17 @@ config() { echo '{"jobsCount":'$jobManagerJobsCount',"topic":"'$job_topic'","bootStrapServer":"'$brokerList'","zookeeperConnect":"'$zookeeper'","consumerGroup":"jobmanager","slackChannel":"#test_channel","slackUserName":"JobManager","tempBucket":"'$bucket'","tempFolder":"'$temp_folder'"}' ;; "wfs") - echo '{"search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"{{ dp_raw_telemetry_backup_location }}","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","store":"{{ cloud_service_provider }}","apiVersion":"v2","parallelization":200},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":200,"appName":"Workflow Summarizer","deviceMapping":true}' + echo '{"search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"{{ dp_raw_telemetry_backup_location }}","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","store":"{{ cloud_service_provider }}","apiVersion":"v2","parallelization":200},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":200,"appName":"Workflow Summarizer","deviceMapping":true}' #echo '{"search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"unique/","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.model.WorkflowSummary","modelParams":{"apiVersion":"v2"},"output":[{"to":"console","params":{"printEvent": false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$topic'"}}],"parallelization":8,"appName":"Workflow Summarizer","deviceMapping":true}' ;; "video-streaming") echo '{"search":{"type":"{{ cloud_service_provider }}"},"model":"org.ekstep.analytics.job.VideoStreamingJob","modelParams":{"maxIterations":10},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Video Streaming Job","deviceMapping":false}' ;; "admin-user-reports") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminReportJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin User Reports","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminReportJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin User Reports","deviceMapping":false}' ;; "admin-geo-reports") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminGeoReportJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin Geo Reports","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.job.report.StateAdminGeoReportJob","modelParams":{"store":"{{cloud_service_provider}}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","fromDate":"$(date --date yesterday '+%Y-%m-%d')","toDate":"$(date --date yesterday '+%Y-%m-%d')","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Admin Geo Reports","deviceMapping":false}' ;; "telemetry-replay") echo '{"search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"'$inputBucket'","endDate":"'$endDate'","delta":0}]},"model":"org.ekstep.analytics.job.EventsReplayJob","modelParams":{},"output":[{"to":"console","params":{"printEvent":false}},{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$sinkTopic'"}}],"parallelization":8,"appName":"TelemetryReplayJob","deviceMapping":false}' @@ -126,22 +126,22 @@ config() { echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.ETBMetricsJob","modelParams":{"reportConfig":{"id":"etb_metrics","metrics":[],"labels":{"date":"Date","identifier":"Textbook ID","name":"Textbook Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","totalQRCodes":"Total number of QR codes","contentLinkedQR":"Number of QR codes with atleast 1 linked content","withoutContentQR":"Number of QR codes with no linked content","withoutContentT1":"Term 1 QR Codes with no linked content","withoutContentT2":"Term 2 QR Codes with no linked content","status":"Textbook Status","totalContentLinked":"Total content linked","totalQRLinked":"Total QR codes linked to content","totalQRNotLinked":"Total number of QR codes with no linked content","leafNodesCount":"Total number of leaf nodes","leafNodeUnlinked":"Number of leaf nodes with no content","l1Name":"Level 1 Name","l2Name":"Level 2 Name","l3Name":"Level 3 Name","l4Name":"Level 4 Name","l5Name":"Level 5 Name","dialcode":"QR Code","sum(scans)":"Total Scans","noOfContent":"Number of contents","nodeType":"Type of Node","term":"Term"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"dialcode_counts.csv","postContainer":"'$reportPostContainer'"}},"dialcodeReportConfig":{"id":"etb_metrics","metrics":[],"labels":{},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":1,"reportPath":"dialcode_counts.csv","rollupAge":"ACADEMIC_YEAR","rollupCol":"Date","rollupRange":10,"postContainer":"'$reportPostContainer'"}},"etbFileConfig":{"bucket":"'$reportPostContainer'","file":"dialcode_scans/dialcode_counts.csv"},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"equals","dimension":"contentType","value":"TextBook"},{"type":"in","dimension":"status","values":["Live","Draft","Review"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"tenantConfig":{"tenantId":"","slugName":""},"store":"{{ cloud_service_provider }}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ETB Metrics Model","deviceMapping":false}' ;; "course-enrollment-report") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseEnrollmentJob","modelParams":{"reportConfig":{"id":"tpd_metrics","metrics":[],"labels":{"completionCount":"Completion Count","status":"Status","enrollmentCount":"Enrollment Count","courseName":"Course Name","batchName":"Batch Name"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"course_enrollment.csv"}},"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]},"limit":10000}},"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Enrollment Metrics Model","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseEnrollmentJob","modelParams":{"reportConfig":{"id":"tpd_metrics","metrics":[],"labels":{"completionCount":"Completion Count","status":"Status","enrollmentCount":"Enrollment Count","courseName":"Course Name","batchName":"Batch Name"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"DAY","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"course_enrollment.csv"}},"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]},"limit":10000}},"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Enrollment Metrics Model","deviceMapping":false}' ;; "course-consumption-report") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseConsumptionJob","modelParams":{"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]}}},"reportConfig":{"id":"tpd_metrics","labels":{"date":"Date","status":"Batch Status","timespent":"Timespent in mins","courseName":"Course Name","batchName":"Batch Name"},"dateRange":{"staticInterval":"LastDay","granularity":"all"},"metrics":[{"metric":"totalCoursePlays","label":"Total Course Plays (in mins)","druidQuery":{"queryType":"groupBy","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"edata_time_spent"}],"dimensions":[{"fieldName":"object_rollup_l1","aliasName":"courseId"},{"fieldName":"uid","aliasName":"userId"},{"fieldName":"context_cdata_id","aliasName":"batchId"}],"filters":[{"type":"equals","dimension":"eid","value":"ME_WORKFLOW_SUMMARY"},{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"context_cdata_type","value":"batch"}],"postAggregation":[{"type":"arithmetic","name":"timespent","fields":{"leftField":"sum__edata_time_spent","rightField":60,"rightFieldType":"constant"},"fn":"/"}],"descending":"false"}}],"output":[{"type":"csv","metrics":["timespent"],"dims":[]}],"queryType":"groupBy"},"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Consumption Metrics Model","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.CourseConsumptionJob","modelParams":{"esConfig":{"request":{"filters":{"objectType":["Content"],"contentType":["Course"],"identifier":[],"status":["Live"]}}},"reportConfig":{"id":"tpd_metrics","labels":{"date":"Date","status":"Batch Status","timespent":"Timespent in mins","courseName":"Course Name","batchName":"Batch Name"},"dateRange":{"staticInterval":"LastDay","granularity":"all"},"metrics":[{"metric":"totalCoursePlays","label":"Total Course Plays (in mins)","druidQuery":{"queryType":"groupBy","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"sum__edata_time_spent","type":"doubleSum","fieldName":"edata_time_spent"}],"dimensions":[{"fieldName":"object_rollup_l1","aliasName":"courseId"},{"fieldName":"uid","aliasName":"userId"},{"fieldName":"context_cdata_id","aliasName":"batchId"}],"filters":[{"type":"equals","dimension":"eid","value":"ME_WORKFLOW_SUMMARY"},{"type":"in","dimension":"dimensions_pdata_id","values":["'$producerEnv'.app","'$producerEnv'.portal"]},{"type":"equals","dimension":"dimensions_type","value":"content"},{"type":"equals","dimension":"dimensions_mode","value":"play"},{"type":"equals","dimension":"context_cdata_type","value":"batch"}],"postAggregation":[{"type":"arithmetic","name":"timespent","fields":{"leftField":"sum__edata_time_spent","rightField":60,"rightFieldType":"constant"},"fn":"/"}],"descending":"false"}}],"output":[{"type":"csv","metrics":["timespent"],"dims":[]}],"queryType":"groupBy"},"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","sparkElasticsearchConnectionHost":"'$sunbirdPlatformElasticsearchHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"TPD Course Consumption Metrics Model","deviceMapping":false}' ;; "textbook-progress-report") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.TextBookProgressModel","modelParams":{"reportConfig":{"id":"content_progress_metrics","metrics":[],"labels":{"board":"Board","medium":"Medium","gradeLevel":"Grade","subject":"Subject","resourceType":"Content Type","totalContent":"Total Contents","live":"Live","review":"Review","draft":"Draft","unlisted":"Limited Sharing","application_ecml":"Created on Diksha","video_youtube":"YouTube Content","video_mp4":"Uploaded Videos","application_pdf":"Text Content","application_html":"Uploaded Interactive Content","identifier":"Content ID","creator":"Created By","createdOn":"Creation Date","lastPublishDate":"Last Publish Date","status":"Status","pkgVersion":"Number of times Published","lastPublishedOn":"Pending in current status since","pendingInCurrentStatus":"Pending in current status since"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"content_progress_metrics.csv","postContainer":"'$reportPostContainer'"}},"filter":{"tenantId":"","slugName":""},"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Progress Metrics Model","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.report.TextBookProgressModel","modelParams":{"reportConfig":{"id":"content_progress_metrics","metrics":[],"labels":{"board":"Board","medium":"Medium","gradeLevel":"Grade","subject":"Subject","resourceType":"Content Type","totalContent":"Total Contents","live":"Live","review":"Review","draft":"Draft","unlisted":"Limited Sharing","application_ecml":"Created on Diksha","video_youtube":"YouTube Content","video_mp4":"Uploaded Videos","application_pdf":"Text Content","application_html":"Uploaded Interactive Content","identifier":"Content ID","creator":"Created By","createdOn":"Creation Date","lastPublishDate":"Last Publish Date","status":"Status","pkgVersion":"Number of times Published","lastPublishedOn":"Pending in current status since","pendingInCurrentStatus":"Pending in current status since"},"output":[{"type":"csv","dims":[]}],"mergeConfig":{"frequency":"WEEK","basePath":"'$baseScriptPath'","rollup":0,"reportPath":"content_progress_metrics.csv","postContainer":"'$reportPostContainer'"}},"filter":{"tenantId":"","slugName":""},"store":"{{ cloud_service_provider }}","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$bucket'","folderPrefix":["slug","reportName"],"sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Progress Metrics Model","deviceMapping":false}' ;; "audit-metrics-report") echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.model.MetricsAuditJob","modelParams":{"auditConfig":[{"name":"denorm","search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"telemetry-denormalized/raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]},"filters":[{"name":"flags.user_data_retrieved","operator":"EQ","value":true},{"name":"flags.content_data_retrieved","operator":"EQ","value":true},{"name":"flags.device_data_retrieved","operator":"EQ","value":true},{"name":"flags.dialcode_data_retrieved","operator":"EQ","value":true},{"name":"flags.collection_data_retrieved","operator":"EQ","value":true},{"name":"flags.derived_location_retrieved","operator":"EQ","value":true}]},{"name":"failed","search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"failed/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"unique","search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"unique/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"raw","search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"raw/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"channel-raw","search":{"type":"{{ cloud_service_provider }}","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/raw/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"channel-summary","search":{"type":"{{ cloud_service_provider }}","queries":[{"folder":true,"bucket":"'$bucket'","prefix":"channel/*/summary/","startDate":"'$endDate'","endDate":"'$endDate'*.json.gz"}]}},{"name":"derived","search":{"type":"{{ cloud_service_provider }}","queries":[{"bucket":"'$bucket'","prefix":"derived/wfs/","startDate":"'$endDate'","endDate":"'$endDate'"}]}},{"name":"telemetry-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"telemetry-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}},{"name":"summary-count","search":{"type":"druid","druidQuery":{"queryType":"timeSeries","dataSource":"summary-events","intervals":"LastDay","aggregations":[{"name":"total_count","type":"count","fieldName":"count"}],"descending":"false"}}}]},"output":[{"to":"kafka","params":{"brokerList":"'$brokerList'","topic":"'$metricsTopic'"}}],"parallelization":8,"appName":"Metrics Audit"}' ;; "sourcing-metrics") - echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.sourcing.SourcingMetrics","modelParams":{"reportConfig":{"id":"textbook_report","metrics":[],"labels":{"date":"Date","primaryCategory":"Collection Category","identifier":"Collection ID","name":"Collection Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","reportDate":"Report generation date","board":"Board","grade":"Grade","chapters":"Folder Name","totalChapters":"Total number of first level folders","status":"Textbook Status"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]},{"type":"json","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}]},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"createdFor","aliasName":"createdFor"},{"fieldName":"createdOn","aliasName":"createdOn"},{"fieldName":"lastUpdatedOn","aliasName":"lastUpdatedOn"},{"fieldName":"board","aliasName":"board"},{"fieldName":"medium","aliasName":"medium"},{"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName":"gradeLevel","aliasName":"gradeLevel"},{"fieldName":"subject","aliasName":"subject"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"in","dimension":"primaryCategory","values":["Digital Textbook","Course","Content Playlist","Question paper","Question Paper"]},{"type":"in","dimension":"status","values":["Live"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"store":"{{ cloud_service_provider }}","storageContainer":"'$reportPostContainer'","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$reportPostContainer'","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Report Job","deviceMapping":false}' + echo '{"search":{"type":"none"},"model":"org.ekstep.analytics.sourcing.SourcingMetrics","modelParams":{"reportConfig":{"id":"textbook_report","metrics":[],"labels":{"date":"Date","primaryCategory":"Collection Category","identifier":"Collection ID","name":"Collection Name","medium":"Medium","gradeLevel":"Grade","subject":"Subject","createdOn":"Created On","lastUpdatedOn":"Last Updated On","reportDate":"Report generation date","board":"Board","grade":"Grade","chapters":"Folder Name","totalChapters":"Total number of first level folders","status":"Textbook Status"},"output":[{"type":"csv","dims":["identifier","channel","name"],"fileParameters":["id","dims"]},{"type":"json","dims":["identifier","channel","name"],"fileParameters":["id","dims"]}]},"druidConfig":{"queryType":"groupBy","dataSource":"content-model-snapshot","intervals":"1901-01-01T00:00:00+00:00/2101-01-01T00:00:00+00:00","aggregations":[{"name":"count","type":"count"}],"dimensions":[{"fieldName":"channel","aliasName":"channel"},{"fieldName":"identifier","aliasName":"identifier","type":"Extraction","outputType":"STRING","extractionFn":[{"type":"javascript","fn":"function(str){return str == null ? null: str.split(\".\")[0]}"}]},{"fieldName":"name","aliasName":"name"},{"fieldName":"createdFor","aliasName":"createdFor"},{"fieldName":"createdOn","aliasName":"createdOn"},{"fieldName":"lastUpdatedOn","aliasName":"lastUpdatedOn"},{"fieldName":"board","aliasName":"board"},{"fieldName":"medium","aliasName":"medium"},{"fieldName":"primaryCategory","aliasName":"primaryCategory"},{"fieldName":"gradeLevel","aliasName":"gradeLevel"},{"fieldName":"subject","aliasName":"subject"},{"fieldName":"status","aliasName":"status"}],"filters":[{"type":"in","dimension":"primaryCategory","values":["Digital Textbook","Course","Content Playlist","Question paper","Question Paper"]},{"type":"in","dimension":"status","values":["Live"]}],"postAggregation":[],"descending":"false","limitSpec":{"type":"default","limit":1000000,"columns":[{"dimension":"count","direction":"descending"}]}},"store":"{{ cloud_service_provider }}","storageContainer":"'$reportPostContainer'","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageEndpoint":"{{cloud_private_storage_endpoint}}","format":"csv","key":"druid-reports/","filePath":"druid-reports/","container":"'$reportPostContainer'","sparkCassandraConnectionHost":"'$sunbirdPlatformCassandraHost'","folderPrefix":["slug","reportName"]},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"Textbook Report Job","deviceMapping":false}' ;; "druid-dataset") - echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.OnDemandDruidExhaustJob","modelParams":{"store":"{{ cloud_service_provider }}","container":"'$reportPostContainer'","key":"ml_reports/","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{dp_storage_endpoint_config}}","format":"csv"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ML Druid Data Model"}' + echo '{"search":{"type":"none"},"model":"org.sunbird.analytics.exhaust.OnDemandDruidExhaustJob","modelParams":{"store":"{{ cloud_service_provider }}","container":"'$reportPostContainer'","key":"ml_reports/","storageKeyConfig":"storage.key.config","storageSecretConfig":"storage.secret.config","storageContainer":"{{reports_container}}","storageEndpoint":"{{cloud_private_storage_endpoint}}","format":"csv"},"output":[{"to":"console","params":{"printEvent":false}}],"parallelization":8,"appName":"ML Druid Data Model"}' ;; "*") echo "Unknown model code" diff --git a/ansible/roles/data-products-deploy/templates/model-config.json.j2 b/ansible/roles/data-products-deploy/templates/model-config.json.j2 index 895b41186d..aa54c3413a 100644 --- a/ansible/roles/data-products-deploy/templates/model-config.json.j2 +++ b/ansible/roles/data-products-deploy/templates/model-config.json.j2 @@ -24,7 +24,7 @@ "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{dp_storage_endpoint_config}}", + "storageEndpoint":"{{cloud_private_storage_endpoint}}", "store":"{{ cloud_service_provider }}" }, "output": [ @@ -76,7 +76,7 @@ "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{dp_storage_endpoint_config}}", + "storageEndpoint":"{{cloud_private_storage_endpoint}}", "store":"{{ cloud_service_provider }}", "sparkCassandraConnectionHost": "{{core_cassandra_host}}", "sparkElasticsearchConnectionHost": "{{sunbird_es_host}}" @@ -102,7 +102,7 @@ "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{dp_storage_endpoint_config}}", + "storageEndpoint":"{{cloud_private_storage_endpoint}}", "store":"{{ cloud_service_provider }}", "sparkCassandraConnectionHost": "{{core_cassandra_host}}", "sparkElasticsearchConnectionHost": "{{sunbird_es_host}}" @@ -129,7 +129,7 @@ "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{dp_storage_endpoint_config}}", + "storageEndpoint":"{{cloud_private_storage_endpoint}}", "startDate": "$(date --date yesterday '+%Y-%m-%d')", "endDate": "$(date '+%Y-%m-%d')" }, @@ -162,7 +162,7 @@ "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{dp_storage_endpoint_config}}", + "storageEndpoint":"{{cloud_private_storage_endpoint}}", "store":"{{ cloud_service_provider }}", "model": [ { @@ -230,7 +230,7 @@ "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{dp_storage_endpoint_config}}", + "storageEndpoint":"{{cloud_private_storage_endpoint}}", "sparkElasticsearchConnectionHost": "{{ lp_composite_search_host }}" }, "output": [ @@ -331,7 +331,7 @@ "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{dp_storage_endpoint_config}}", + "storageEndpoint":"{{cloud_private_storage_endpoint}}", "format": "csv", "key": "druid-reports/", "filePath": "druid-reports/", @@ -384,7 +384,7 @@ "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{dp_storage_endpoint_config}}", + "storageEndpoint":"{{cloud_private_storage_endpoint}}", "format":"csv", "key": "druid-reports/", "filePath": "druid-reports/", @@ -499,7 +499,7 @@ "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{dp_storage_endpoint_config}}", + "storageEndpoint":"{{cloud_private_storage_endpoint}}", "format":"csv", "key": "druid-reports/", "filePath": "druid-reports/", @@ -528,7 +528,7 @@ "storageKeyConfig":"storage.key.config", "storageSecretConfig":"storage.secret.config", "storageContainer":"{{reports_container}}", - "storageEndpoint":"{{dp_storage_endpoint_config}}", + "storageEndpoint":"{{cloud_private_storage_endpoint}}", "auditConfig": [ { "name": "denorm", From 77192468e3811f71a5aa9b1c801540c1a7f39d17 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Mon, 4 Dec 2023 13:09:09 +0530 Subject: [PATCH 16/46] #OB-553 fix: Deployment issue fixes for OCI --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 4e2920caf7..2d4a9215e1 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -6,8 +6,8 @@ image_tag: {{ image_tag }} checkpoint_store_type: {{ checkpoint_store_type }} azure_account: {{ azure_account }} azure_secret: {{ azure_secret }} -s3_access_key: {{ s3_storage_key }} -s3_secret_key: {{ s3_storage_secret }} +s3_access_key: {{ cloud_private_storage_accountname }} +s3_secret_key: {{ cloud_private_storage_secret }} {% if cloud_service_provider == "oci" %} s3_endpoint: {{ oci_flink_s3_storage_endpoint }} {% else %} From 79274336c8ea3fea5a43ffdc8c68b51643de3d52 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Mon, 4 Dec 2023 13:14:04 +0530 Subject: [PATCH 17/46] #OB-553 fix: Deployment issue fixes for OCI --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 2d4a9215e1..e2870a4f0b 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -9,7 +9,7 @@ azure_secret: {{ azure_secret }} s3_access_key: {{ cloud_private_storage_accountname }} s3_secret_key: {{ cloud_private_storage_secret }} {% if cloud_service_provider == "oci" %} -s3_endpoint: {{ oci_flink_s3_storage_endpoint }} +s3_endpoint: {{ cloud_private_storage_endpoint }} {% else %} s3_endpoint: {{ s3_storage_endpoint }} {% endif %} From a558861dcb9ff02fecc071b76c0829a570579340 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Mon, 4 Dec 2023 13:22:15 +0530 Subject: [PATCH 18/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/inventory/env/group_vars/all.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index d11ccd88f0..303587c85c 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -85,6 +85,7 @@ postgres: db_port: 5432 db_admin_user: analytics db_admin_password: "{{dp_vault_pgdb_admin_password}}" + dp_ssl: "require" postgres_address_space: 0.0.0.0/0 # Postgres trust address space From cd1fac68bcf22613a634a6d523f90bf693445fff Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Mon, 4 Dec 2023 16:51:27 +0530 Subject: [PATCH 19/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/inventory/env/group_vars/all.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index 303587c85c..6e4f5cf267 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -85,7 +85,8 @@ postgres: db_port: 5432 db_admin_user: analytics db_admin_password: "{{dp_vault_pgdb_admin_password}}" - dp_ssl: "require" + dp_ssl: "" + dp_db_name: analytics postgres_address_space: 0.0.0.0/0 # Postgres trust address space From 5b2ad02d6c476dc9001f955b747eff9a6f7aefb0 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 7 Dec 2023 12:25:37 +0530 Subject: [PATCH 20/46] #OB-553 fix: Deployment issue fixes for OCI --- kubernetes/helm_charts/secor/templates/secor-config.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/kubernetes/helm_charts/secor/templates/secor-config.yaml b/kubernetes/helm_charts/secor/templates/secor-config.yaml index f3d9cf3a55..4cec5c8b98 100644 --- a/kubernetes/helm_charts/secor/templates/secor-config.yaml +++ b/kubernetes/helm_charts/secor/templates/secor-config.yaml @@ -16,6 +16,8 @@ data: {{- (tpl ($.Files.Get "config/secor.common.properties") $) | nindent 4 }} secor.azure.properties: |+ {{- (tpl ($.Files.Get "config/secor.azure.properties") $) | nindent 4 }} + secor.s3.properties: |+ +{{- (tpl ($.Files.Get "config/secor.s3.properties") $) | nindent 4 }} secor.partition.properties: |+ {{- (tpl ($.Files.Get "config/secor.partition.properties") $) | nindent 4 }} log4j.properties: |+ From 9493def828dbda05b423b1f20e0ad4829983d8f3 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Mon, 11 Dec 2023 17:49:53 +0530 Subject: [PATCH 21/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/roles/data-products-deploy/templates/common.conf.j2 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ansible/roles/data-products-deploy/templates/common.conf.j2 b/ansible/roles/data-products-deploy/templates/common.conf.j2 index 4484e19641..63691ab9bf 100644 --- a/ansible/roles/data-products-deploy/templates/common.conf.j2 +++ b/ansible/roles/data-products-deploy/templates/common.conf.j2 @@ -8,6 +8,10 @@ service.search.path="{{ service.search.path }}" spark.cassandra.connection.host="{{groups['dp-cassandra'][0]}}" cassandra.keyspace_prefix="{{ cassandra_keyspace_prefix }}" cassandra.hierarchy_store_prefix="{{ cassandra_hierarchy_store_prefix }}" + +cloud_storage_type="{{ cloud_service_provider }}" +cloud_storage_endpoint_with_protocol="{{ cloud_private_storage_endpoint }}" + storage.key.config="{{ dp_storage_key_config }}" storage.secret.config="{{ dp_storage_secret_config }}" reports.storage.key.config="{{ dp_reports_storage_key_config }}" From 72d858c03ef0f4e6af81d15702b1c9ce5bc8b2a6 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 12 Dec 2023 14:39:40 +0530 Subject: [PATCH 22/46] #OB-553 fix: Deployment issue fixes for OCI --- kubernetes/helm_charts/secor/values.j2 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/kubernetes/helm_charts/secor/values.j2 b/kubernetes/helm_charts/secor/values.j2 index 7d9aa8ecec..1504a101fb 100644 --- a/kubernetes/helm_charts/secor/values.j2 +++ b/kubernetes/helm_charts/secor/values.j2 @@ -2,10 +2,10 @@ azure_account: "{{ cloud_private_storage_accountname }}" azure_secret: "{{ cloud_private_storage_secret }}" azure_container_name: "telemetry-data-store" -s3_access_key: "{{s3_storage_key}}" -s3_secret_id: "{{s3_storage_secret}}" -s3_region: "{{oci_region}}" -s3_endpoint: "{{s3_storage_endpoint}}" +s3_access_key: "{{cloud_private_storage_accountname}}" +s3_secret_id: "{{cloud_private_storage_secret}}" +s3_region: "{{cloud_private_storage_region}}" +s3_endpoint: "{{cloud_private_storage_endpoint}}" s3_path_style_access: "{{s3_path_style_access}}" s3_bucket_name: "telemetry-data-store" From 11489239cbd11b8bb9a252043c1078c9c16c0a5d Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 12 Dec 2023 15:03:13 +0530 Subject: [PATCH 23/46] #OB-553 fix: Deployment issue fixes for OCI - Secor --- kubernetes/ansible/roles/secor-deploy/defaults/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml b/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml index 85d6f0f625..d47ac250a5 100644 --- a/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml +++ b/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml @@ -8,7 +8,7 @@ secor_storage_class: managed-premium # this varible is to control the monitoring deployment. override this value by adding into # private repo common.yml to not to create the secor lag alert rules. The scenorio is , if adopters does not have # monitoring stack then the secor deployment job will fail. To avoid that we introduced this var. -secor_alertrule_enabled: true +secor_alertrule_enabled: false # image: # prefix: {} # repository: "secor" From f84ee0275a1320095e12a703a3afb58b904940d9 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 12 Dec 2023 15:11:03 +0530 Subject: [PATCH 24/46] #OB-553 fix: Deployment issue fixes for OCI - Secor --- kubernetes/ansible/roles/secor-deploy/tasks/main.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml b/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml index 68778084ee..2d13045d07 100644 --- a/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml +++ b/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml @@ -11,8 +11,8 @@ - "{{ job_names_to_deploy.split(',')|list }}" tags: secor_deploy -- name: create alert rules - shell: helm upgrade --install {{ item }} {{ chart_path }}/charts/alertrules -f {{ chart_path }}/values.yaml - with_items: - - "{{ job_names_to_deploy.split(',')|list }}" - tags: alertrule +# - name: create alert rules +# shell: helm upgrade --install {{ item }} {{ chart_path }}/charts/alertrules -f {{ chart_path }}/values.yaml +# with_items: +# - "{{ job_names_to_deploy.split(',')|list }}" +# tags: alertrule From 4c85ce95a15add1c948a683da20305a711e716c6 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 12 Dec 2023 15:37:18 +0530 Subject: [PATCH 25/46] #OB-553 fix: Deployment issue fixes for OCI - Secor --- kubernetes/helm_charts/secor/config/secor.common.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/secor/config/secor.common.properties b/kubernetes/helm_charts/secor/config/secor.common.properties index fc207e660c..7428fa9b7b 100644 --- a/kubernetes/helm_charts/secor/config/secor.common.properties +++ b/kubernetes/helm_charts/secor/config/secor.common.properties @@ -358,7 +358,7 @@ secor.max.message.size.bytes=100000 # Class that will manage uploads. Default is to use the hadoop # interface to S3. # secor.upload.manager.class=com.pinterest.secor.uploader.AzureUploadManager -secor.upload.manager.class= +#secor.upload.manager.class=com.pinterest.secor.uploader.S3UploadManager #Set below property to your timezone, and the events will be parsed and converted to the timezone specified secor.message.timezone=UTC From 9d9f2334cfa4e9dd6a7a61756274dd142ae8eb6e Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 12 Dec 2023 15:50:08 +0530 Subject: [PATCH 26/46] #OB-553 fix: Deployment issue fixes for OCI - Secor --- kubernetes/helm_charts/secor/values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/secor/values.j2 b/kubernetes/helm_charts/secor/values.j2 index 1504a101fb..2d8148e53c 100644 --- a/kubernetes/helm_charts/secor/values.j2 +++ b/kubernetes/helm_charts/secor/values.j2 @@ -6,7 +6,7 @@ s3_access_key: "{{cloud_private_storage_accountname}}" s3_secret_id: "{{cloud_private_storage_secret}}" s3_region: "{{cloud_private_storage_region}}" s3_endpoint: "{{cloud_private_storage_endpoint}}" -s3_path_style_access: "{{s3_path_style_access}}" +s3_path_style_access: "false" s3_bucket_name: "telemetry-data-store" {% if cloud_service_provider == 'oci' -%} From 90fcbe19a5f908f1d227014a535f3a978ae8d3b7 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 12 Dec 2023 15:58:14 +0530 Subject: [PATCH 27/46] #OB-553 fix: Deployment issue fixes for OCI - Secor --- kubernetes/helm_charts/secor/config/secor.common.properties | 2 +- kubernetes/helm_charts/secor/values.j2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/helm_charts/secor/config/secor.common.properties b/kubernetes/helm_charts/secor/config/secor.common.properties index 7428fa9b7b..2ae318da2c 100644 --- a/kubernetes/helm_charts/secor/config/secor.common.properties +++ b/kubernetes/helm_charts/secor/config/secor.common.properties @@ -56,7 +56,7 @@ aws.endpoint= # Toggle the AWS S3 client between virtual host style access and path style # access. See http://docs.aws.amazon.com/AmazonS3/latest/dev/VirtualHosting.html -aws.client.pathstyleaccess= +#aws.client.pathstyleaccess= ########################### # START AWS S3 ENCRYPTION # diff --git a/kubernetes/helm_charts/secor/values.j2 b/kubernetes/helm_charts/secor/values.j2 index 2d8148e53c..1504a101fb 100644 --- a/kubernetes/helm_charts/secor/values.j2 +++ b/kubernetes/helm_charts/secor/values.j2 @@ -6,7 +6,7 @@ s3_access_key: "{{cloud_private_storage_accountname}}" s3_secret_id: "{{cloud_private_storage_secret}}" s3_region: "{{cloud_private_storage_region}}" s3_endpoint: "{{cloud_private_storage_endpoint}}" -s3_path_style_access: "false" +s3_path_style_access: "{{s3_path_style_access}}" s3_bucket_name: "telemetry-data-store" {% if cloud_service_provider == 'oci' -%} From 638b54d29e9703027d080e2db9d6d95186e815dd Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Tue, 12 Dec 2023 16:06:23 +0530 Subject: [PATCH 28/46] #OB-553 fix: Deployment issue fixes for OCI - Secor --- .../ansible/roles/secor-deploy/defaults/main.yaml | 2 +- kubernetes/ansible/roles/secor-deploy/tasks/main.yaml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml b/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml index d47ac250a5..85d6f0f625 100644 --- a/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml +++ b/kubernetes/ansible/roles/secor-deploy/defaults/main.yaml @@ -8,7 +8,7 @@ secor_storage_class: managed-premium # this varible is to control the monitoring deployment. override this value by adding into # private repo common.yml to not to create the secor lag alert rules. The scenorio is , if adopters does not have # monitoring stack then the secor deployment job will fail. To avoid that we introduced this var. -secor_alertrule_enabled: false +secor_alertrule_enabled: true # image: # prefix: {} # repository: "secor" diff --git a/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml b/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml index 2d13045d07..68778084ee 100644 --- a/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml +++ b/kubernetes/ansible/roles/secor-deploy/tasks/main.yaml @@ -11,8 +11,8 @@ - "{{ job_names_to_deploy.split(',')|list }}" tags: secor_deploy -# - name: create alert rules -# shell: helm upgrade --install {{ item }} {{ chart_path }}/charts/alertrules -f {{ chart_path }}/values.yaml -# with_items: -# - "{{ job_names_to_deploy.split(',')|list }}" -# tags: alertrule +- name: create alert rules + shell: helm upgrade --install {{ item }} {{ chart_path }}/charts/alertrules -f {{ chart_path }}/values.yaml + with_items: + - "{{ job_names_to_deploy.split(',')|list }}" + tags: alertrule From 44a167b69a1237a8b6fc5fe163d589a6e092fddb Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 13:15:17 +0530 Subject: [PATCH 29/46] #OB-553 fix: Deployment issue fixes for OCI --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index e2870a4f0b..339e863f83 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -3,7 +3,7 @@ imagepullsecrets: {{ imagepullsecrets }} dockerhub: {{ dockerhub }} repository: {{flink_repository|default('sunbird-datapipeline')}} image_tag: {{ image_tag }} -checkpoint_store_type: {{ checkpoint_store_type }} +checkpoint_store_type: {{ cloud_service_provider }} azure_account: {{ azure_account }} azure_secret: {{ azure_secret }} s3_access_key: {{ cloud_private_storage_accountname }} From 5eb55a9c9719736a691568b7acd919b5968f9537 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 13:45:47 +0530 Subject: [PATCH 30/46] #OB-553 fix: Deployment issue fixes for OCI --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 17bd0257bb..3524203c0b 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -34,7 +34,7 @@ azure_account: "{{ cloud_private_storage_accountname }}" azure_secret: "{{ cloud_private_storage_secret }}" cloud_storage_flink_bucketname: dev-data-store flink_dp_storage_container: "" -checkpoint_store_type: gcloud +checkpoint_store_type: {{ cloud_service_provider }} checkpoint_interval: 60000 checkpoint_pause_between_seconds: 5000 checkpoint_compression_enabled: true From b0a8e267f80b3ba3eabcc8d0d967c1e59fa92cc6 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 13:54:14 +0530 Subject: [PATCH 31/46] #OB-553 fix: Deployment issue fixes for OCI --- .../kubernetes/helm_charts/datapipeline/flink-jobs/values.j2 | 2 +- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2 b/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2 index d7c53a44c7..80eb60be0d 100644 --- a/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2 +++ b/ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2 @@ -6,7 +6,7 @@ dockerhub: {{ dockerhub }} repository: {{ datapipeline_repository|default('data-pipeline') }} image_tag: {{ image_tag }} -checkpoint_store_type: {{ cloud_service_provider }} +checkpoint_store_type: {{ druid_storage_type }} cloud_storage_key: {{ cloud_private_storage_accountname }} cloud_storage_secret: {{ cloud_private_storage_secret }} cloud_storage_endpoint: {{ cloud_private_storage_endpoint }} diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 3524203c0b..67adce709e 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -34,7 +34,7 @@ azure_account: "{{ cloud_private_storage_accountname }}" azure_secret: "{{ cloud_private_storage_secret }}" cloud_storage_flink_bucketname: dev-data-store flink_dp_storage_container: "" -checkpoint_store_type: {{ cloud_service_provider }} +checkpoint_store_type: {{ druid_storage_type }} checkpoint_interval: 60000 checkpoint_pause_between_seconds: 5000 checkpoint_compression_enabled: true From eeb7318a597470906c24125f8d333d7d1f860cbf Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 13:56:35 +0530 Subject: [PATCH 32/46] #OB-553 fix: Deployment issue fixes for OCI --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 67adce709e..1944419946 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -34,7 +34,7 @@ azure_account: "{{ cloud_private_storage_accountname }}" azure_secret: "{{ cloud_private_storage_secret }}" cloud_storage_flink_bucketname: dev-data-store flink_dp_storage_container: "" -checkpoint_store_type: {{ druid_storage_type }} +checkpoint_store_type: "{{ druid_storage_type }}" checkpoint_interval: 60000 checkpoint_pause_between_seconds: 5000 checkpoint_compression_enabled: true From 983f7c0bde7edb302ee22accab569b913843d7a7 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 14:02:18 +0530 Subject: [PATCH 33/46] #OB-553 fix: Deployment issue fixes for OCI --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 339e863f83..388eaad8c2 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -162,7 +162,7 @@ base_config: | {% if checkpoint_store_type == "azure" %} base.url = "wasbs://"${job.statebackend.blob.storage.container}"@"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.checkpointing.dir} {% elif checkpoint_store_type == "s3" %} - base.url = "s3://"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir} + base.url = "s3://"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir} {% elif checkpoint_store_type == "gcloud" %} base.url = "gs://"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir} {% endif %} From 9cdb8a188a689e728ab65a1d2d09edd703ec98ca Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 14:12:39 +0530 Subject: [PATCH 34/46] #OB-553 fix: Deployment issue fixes for OCI --- .../templates/flink_job_deployment.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index 35d9066161..0928c95786 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -111,17 +111,17 @@ spec: args: ["start-foreground", "--job-classname={{ .Values.job_classname }}", {{- if eq .Values.checkpoint_store_type "azure" }} - "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net={{ .Values.cloud_storage_secret }}", + "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", {{- end }} {{- if eq .Values.checkpoint_store_type "s3" }} - "-Ds3.access-key={{ .Values.cloud_storage_key }}", - "-Ds3.secret-key={{ .Values.cloud_storage_secret }}", + "-Ds3.access-key={{ .Values.cloud_private_storage_accountname }}", + "-Ds3.secret-key={{ .Values.cloud_private_storage_secret }}", "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}", "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", {{- end }} {{- if eq .Values.checkpoint_store_type "gcloud" }} - "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}", - "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}", + "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", + "-Dfs.gs.auth.client.secret={{ .Values.cloud_private_storage_secret }}", "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", {{- end }} "-Dweb.submit.enable=false", @@ -197,17 +197,17 @@ spec: command: ["/opt/flink/bin/taskmanager.sh"] args: ["start-foreground", {{- if eq .Values.checkpoint_store_type "azure" }} - "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net={{ .Values.cloud_storage_secret }}", + "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", {{- end }} {{- if eq .Values.checkpoint_store_type "s3" }} - "-Ds3.access-key={{ .Values.cloud_storage_key }}", - "-Ds3.secret-key={{ .Values.cloud_storage_secret }}", + "-Ds3.access-key={{ .Values.cloud_private_storage_accountname }}", + "-Ds3.secret-key={{ .Values.cloud_private_storage_secret }}", "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}", "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", {{- end }} {{- if eq .Values.checkpoint_store_type "gcloud" }} - "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}", - "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}", + "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", + "-Dfs.gs.auth.client.secret={{ .Values.cloud_private_storage_secret }}", "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", {{- end }} "-Dweb.submit.enable=false", From fac57960e42d0a8e25a6c71575c748b652b3a197 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 14:23:20 +0530 Subject: [PATCH 35/46] #OB-553 fix: Deployment issue fixes for OCI --- .../templates/flink_job_deployment.yaml | 16 ++++++++-------- .../helm_charts/datapipeline_jobs/values.j2 | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index 0928c95786..bcbdd7e954 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -114,10 +114,10 @@ spec: "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", {{- end }} {{- if eq .Values.checkpoint_store_type "s3" }} - "-Ds3.access-key={{ .Values.cloud_private_storage_accountname }}", - "-Ds3.secret-key={{ .Values.cloud_private_storage_secret }}", - "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}", - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", + "-Dpresto.s3.access-key={{ .Values.cloud_private_storage_accountname }}", + "-Dpresto.s3.secret-key={{ .Values.cloud_private_storage_secret }}", + "-Dpresto.s3.endpoint={{ .Values.cloud_storage_endpoint }}", + "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}", {{- end }} {{- if eq .Values.checkpoint_store_type "gcloud" }} "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", @@ -200,10 +200,10 @@ spec: "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", {{- end }} {{- if eq .Values.checkpoint_store_type "s3" }} - "-Ds3.access-key={{ .Values.cloud_private_storage_accountname }}", - "-Ds3.secret-key={{ .Values.cloud_private_storage_secret }}", - "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}", - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", + "-Dpresto.s3.access-key={{ .Values.cloud_private_storage_accountname }}", + "-Dpresto.s3.secret-key={{ .Values.cloud_private_storage_secret }}", + "-Dpresto.s3.endpoint={{ .Values.cloud_storage_endpoint }}", + "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}", {{- end }} {{- if eq .Values.checkpoint_store_type "gcloud" }} "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 388eaad8c2..428ce9c7ec 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -154,7 +154,7 @@ base_config: | statebackend { blob { storage { - account = "{% if checkpoint_store_type == "azure" %}{{ azure_account }}.blob.core.windows.net{% elif checkpoint_store_type == "s3" %}{{ flink_dp_storage_container }}{% elif checkpoint_store_type == "gcloud" %}{{ flink_dp_storage_container }}{% endif %}" + account = "{% if checkpoint_store_type == "azure" %}{{ azure_account }}.blob.core.windows.net{% elif checkpoint_store_type == "s3" %}{{ cloud_private_storage_accountname }}{% elif checkpoint_store_type == "gcloud" %}{{ flink_dp_storage_container }}{% endif %}" container = "{{ cloud_storage_flink_bucketname }}" checkpointing.dir = "checkpoint" } From e0c98b08fb71f9d5ae0dc4d5126725132aa2a671 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 14:32:24 +0530 Subject: [PATCH 36/46] #OB-553 fix: Deployment issue fixes for OCI --- .../templates/flink_job_deployment.yaml | 40 +++++++++---------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index bcbdd7e954..958c0952db 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -110,20 +110,20 @@ spec: command: ["/opt/flink/bin/standalone-job.sh"] args: ["start-foreground", "--job-classname={{ .Values.job_classname }}", -{{- if eq .Values.checkpoint_store_type "azure" }} + {{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", -{{- end }} -{{- if eq .Values.checkpoint_store_type "s3" }} - "-Dpresto.s3.access-key={{ .Values.cloud_private_storage_accountname }}", - "-Dpresto.s3.secret-key={{ .Values.cloud_private_storage_secret }}", - "-Dpresto.s3.endpoint={{ .Values.cloud_storage_endpoint }}", - "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}", -{{- end }} -{{- if eq .Values.checkpoint_store_type "gcloud" }} + {{- end }} + {{- if eq .Values.checkpoint_store_type "s3" }} + "-Ds3.access-key={{ .Values.cloud_private_storage_accountname }}", + "-Ds3.secret-key={{ .Values.cloud_private_storage_secret }}", + "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}", + "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", + {{- end }} + {{- if eq .Values.checkpoint_store_type "gcloud" }} "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", "-Dfs.gs.auth.client.secret={{ .Values.cloud_private_storage_secret }}", "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", -{{- end }} + {{- end }} "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}", @@ -196,20 +196,20 @@ spec: workingDir: {{ .Values.taskmanager.flink_work_dir }} command: ["/opt/flink/bin/taskmanager.sh"] args: ["start-foreground", -{{- if eq .Values.checkpoint_store_type "azure" }} + {{- if eq .Values.checkpoint_store_type "azure" }} "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", -{{- end }} -{{- if eq .Values.checkpoint_store_type "s3" }} - "-Dpresto.s3.access-key={{ .Values.cloud_private_storage_accountname }}", - "-Dpresto.s3.secret-key={{ .Values.cloud_private_storage_secret }}", - "-Dpresto.s3.endpoint={{ .Values.cloud_storage_endpoint }}", - "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}", -{{- end }} -{{- if eq .Values.checkpoint_store_type "gcloud" }} + {{- end }} + {{- if eq .Values.checkpoint_store_type "s3" }} + "-Ds3.access-key={{ .Values.cloud_private_storage_accountname }}", + "-Ds3.secret-key={{ .Values.cloud_private_storage_secret }}", + "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}", + "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", + {{- end }} + {{- if eq .Values.checkpoint_store_type "gcloud" }} "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", "-Dfs.gs.auth.client.secret={{ .Values.cloud_private_storage_secret }}", "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", -{{- end }} + {{- end }} "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager", From 224a28d924d28fa0a606b193a65185286a05e34a Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 14:49:03 +0530 Subject: [PATCH 37/46] #OB-553 fix: Deployment issue fixes for OCI --- .../templates/flink_job_deployment.yaml | 42 +++++++++---------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index 958c0952db..583fee53fa 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -109,21 +109,21 @@ spec: workingDir: /opt/flink command: ["/opt/flink/bin/standalone-job.sh"] args: ["start-foreground", - "--job-classname={{ .Values.job_classname }}", - {{- if eq .Values.checkpoint_store_type "azure" }} - "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", - {{- end }} - {{- if eq .Values.checkpoint_store_type "s3" }} + # {{- if eq .Values.checkpoint_store_type "azure" }} + # "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", + # {{- end }} + # {{- if eq .Values.checkpoint_store_type "s3" }} "-Ds3.access-key={{ .Values.cloud_private_storage_accountname }}", "-Ds3.secret-key={{ .Values.cloud_private_storage_secret }}", "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}", "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", - {{- end }} - {{- if eq .Values.checkpoint_store_type "gcloud" }} - "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", - "-Dfs.gs.auth.client.secret={{ .Values.cloud_private_storage_secret }}", - "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", - {{- end }} + # {{- end }} + # {{- if eq .Values.checkpoint_store_type "gcloud" }} + # "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", + # "-Dfs.gs.auth.client.secret={{ .Values.cloud_private_storage_secret }}", + # "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", + # {{- end }} + "--job-classname={{ .Values.job_classname }}", "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}", @@ -196,20 +196,20 @@ spec: workingDir: {{ .Values.taskmanager.flink_work_dir }} command: ["/opt/flink/bin/taskmanager.sh"] args: ["start-foreground", - {{- if eq .Values.checkpoint_store_type "azure" }} - "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", - {{- end }} - {{- if eq .Values.checkpoint_store_type "s3" }} + # {{- if eq .Values.checkpoint_store_type "azure" }} + # "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", + # {{- end }} + # {{- if eq .Values.checkpoint_store_type "s3" }} "-Ds3.access-key={{ .Values.cloud_private_storage_accountname }}", "-Ds3.secret-key={{ .Values.cloud_private_storage_secret }}", "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}", "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", - {{- end }} - {{- if eq .Values.checkpoint_store_type "gcloud" }} - "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", - "-Dfs.gs.auth.client.secret={{ .Values.cloud_private_storage_secret }}", - "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", - {{- end }} + # {{- end }} + # {{- if eq .Values.checkpoint_store_type "gcloud" }} + # "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", + # "-Dfs.gs.auth.client.secret={{ .Values.cloud_private_storage_secret }}", + # "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", + # {{- end }} "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager", From 8d9bf15c4daa6b9cc756a34c5bfa558ea57e3442 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 15:04:20 +0530 Subject: [PATCH 38/46] #OB-553 fix: Deployment issue fixes for OCI --- kubernetes/helm_charts/datapipeline_jobs/values.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/values.j2 b/kubernetes/helm_charts/datapipeline_jobs/values.j2 index 428ce9c7ec..f591e52019 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/values.j2 +++ b/kubernetes/helm_charts/datapipeline_jobs/values.j2 @@ -3,7 +3,7 @@ imagepullsecrets: {{ imagepullsecrets }} dockerhub: {{ dockerhub }} repository: {{flink_repository|default('sunbird-datapipeline')}} image_tag: {{ image_tag }} -checkpoint_store_type: {{ cloud_service_provider }} +checkpoint_store_type: {{ druid_storage_type }} azure_account: {{ azure_account }} azure_secret: {{ azure_secret }} s3_access_key: {{ cloud_private_storage_accountname }} From eea48edf5634f9f3864d5417df3e4b55ba516802 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 15:09:44 +0530 Subject: [PATCH 39/46] #OB-553 fix: Deployment issue fixes for OCI --- .../templates/flink_job_deployment.yaml | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index 583fee53fa..46dee439f4 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -109,21 +109,21 @@ spec: workingDir: /opt/flink command: ["/opt/flink/bin/standalone-job.sh"] args: ["start-foreground", - # {{- if eq .Values.checkpoint_store_type "azure" }} - # "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", - # {{- end }} - # {{- if eq .Values.checkpoint_store_type "s3" }} - "-Ds3.access-key={{ .Values.cloud_private_storage_accountname }}", - "-Ds3.secret-key={{ .Values.cloud_private_storage_secret }}", - "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}", - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", - # {{- end }} - # {{- if eq .Values.checkpoint_store_type "gcloud" }} - # "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", - # "-Dfs.gs.auth.client.secret={{ .Values.cloud_private_storage_secret }}", - # "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", - # {{- end }} "--job-classname={{ .Values.job_classname }}", + {{- if eq .Values.checkpoint_store_type "azure" }} + "-Dfs.azure.account.key.{{ cloud_private_storage_accountname }}.blob.core.windows.net={{ cloud_private_storage_secret }}", + {{- end }} + {{- if eq .Values.checkpoint_store_type "s3" }} + "-Ds3.access-key={{ cloud_private_storage_accountname }}", + "-Ds3.secret-key={{ cloud_private_storage_secret }}", + "-Ds3.endpoint={{ cloud_storage_endpoint }}", + "-Ds3.path.style.access={{ s3_path_style_access }}", + {{- end }} + {{- if eq .Values.checkpoint_store_type "gcloud" }} + "-Dfs.gs.auth.client.id={{ cloud_private_storage_accountname }}", + "-Dfs.gs.auth.client.secret={{ cloud_private_storage_secret }}", + "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", + {{- end }} "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}", @@ -196,26 +196,26 @@ spec: workingDir: {{ .Values.taskmanager.flink_work_dir }} command: ["/opt/flink/bin/taskmanager.sh"] args: ["start-foreground", - # {{- if eq .Values.checkpoint_store_type "azure" }} - # "-Dfs.azure.account.key.{{ .Values.cloud_private_storage_accountname }}.blob.core.windows.net={{ .Values.cloud_private_storage_secret }}", - # {{- end }} - # {{- if eq .Values.checkpoint_store_type "s3" }} - "-Ds3.access-key={{ .Values.cloud_private_storage_accountname }}", - "-Ds3.secret-key={{ .Values.cloud_private_storage_secret }}", - "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}", - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", - # {{- end }} - # {{- if eq .Values.checkpoint_store_type "gcloud" }} - # "-Dfs.gs.auth.client.id={{ .Values.cloud_private_storage_accountname }}", - # "-Dfs.gs.auth.client.secret={{ .Values.cloud_private_storage_secret }}", - # "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", - # {{- end }} - "-Dweb.submit.enable=false", - "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", - "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager", - "-Dmetrics.reporter.prom.port=9251-9260", - "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager", - "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}"] + {{- if eq .Values.checkpoint_store_type "azure" }} + "-Dfs.azure.account.key.{{ cloud_private_storage_accountname }}.blob.core.windows.net={{ cloud_private_storage_secret }}", + {{- end }} + {{- if eq .Values.checkpoint_store_type "s3" }} + "-Ds3.access-key={{ cloud_private_storage_accountname }}", + "-Ds3.secret-key={{ cloud_private_storage_secret }}", + "-Ds3.endpoint={{ cloud_storage_endpoint }}", + "-Ds3.path.style.access={{ s3_path_style_access }}", + {{- end }} + {{- if eq .Values.checkpoint_store_type "gcloud" }} + "-Dfs.gs.auth.client.id={{ cloud_private_storage_accountname }}", + "-Dfs.gs.auth.client.secret={{ cloud_private_storage_secret }}", + "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", + {{- end }} + "-Dweb.submit.enable=false", + "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", + "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager", + "-Dmetrics.reporter.prom.port=9251-9260", + "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager", + "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}"] ports: - containerPort: {{ .Values.taskmanager.rpc_port }} name: rpc From 38fa220030dff8d5f2f7b64ef8ba6e9147c6a9b8 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 15:16:27 +0530 Subject: [PATCH 40/46] #OB-553 fix: Deployment issue fixes for OCI --- .../templates/flink_job_deployment.yaml | 52 +++++++++---------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index 46dee439f4..b85c7dd9ec 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -111,19 +111,19 @@ spec: args: ["start-foreground", "--job-classname={{ .Values.job_classname }}", {{- if eq .Values.checkpoint_store_type "azure" }} - "-Dfs.azure.account.key.{{ cloud_private_storage_accountname }}.blob.core.windows.net={{ cloud_private_storage_secret }}", + "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}", {{- end }} {{- if eq .Values.checkpoint_store_type "s3" }} - "-Ds3.access-key={{ cloud_private_storage_accountname }}", - "-Ds3.secret-key={{ cloud_private_storage_secret }}", - "-Ds3.endpoint={{ cloud_storage_endpoint }}", - "-Ds3.path.style.access={{ s3_path_style_access }}", - {{- end }} - {{- if eq .Values.checkpoint_store_type "gcloud" }} - "-Dfs.gs.auth.client.id={{ cloud_private_storage_accountname }}", - "-Dfs.gs.auth.client.secret={{ cloud_private_storage_secret }}", - "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", + "-Ds3.access-key={{ .Values.s3_access_key }}", + "-Ds3.secret-key={{ .Values.s3_secret_key }}", + "-Ds3.endpoint={{ .Values.s3_endpoint }}", + "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", {{- end }} + # {{- if eq .Values.checkpoint_store_type "gcloud" }} + # "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}", + # "-Dfs.gs.auth.client.secret={{ .Values.s3_access_key }}", + # "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", + # {{- end }} "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}", @@ -197,25 +197,25 @@ spec: command: ["/opt/flink/bin/taskmanager.sh"] args: ["start-foreground", {{- if eq .Values.checkpoint_store_type "azure" }} - "-Dfs.azure.account.key.{{ cloud_private_storage_accountname }}.blob.core.windows.net={{ cloud_private_storage_secret }}", + "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}", {{- end }} {{- if eq .Values.checkpoint_store_type "s3" }} - "-Ds3.access-key={{ cloud_private_storage_accountname }}", - "-Ds3.secret-key={{ cloud_private_storage_secret }}", - "-Ds3.endpoint={{ cloud_storage_endpoint }}", - "-Ds3.path.style.access={{ s3_path_style_access }}", - {{- end }} - {{- if eq .Values.checkpoint_store_type "gcloud" }} - "-Dfs.gs.auth.client.id={{ cloud_private_storage_accountname }}", - "-Dfs.gs.auth.client.secret={{ cloud_private_storage_secret }}", - "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", + "-Ds3.access-key={{ .Values.s3_access_key }}", + "-Ds3.secret-key={{ .Values.s3_secret_key }}", + "-Ds3.endpoint={{ .Values.s3_endpoint }}", + "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", {{- end }} - "-Dweb.submit.enable=false", - "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", - "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager", - "-Dmetrics.reporter.prom.port=9251-9260", - "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager", - "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}"] + # {{- if eq .Values.checkpoint_store_type "gcloud" }} + # "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}", + # "-Dfs.gs.auth.client.secret={{ .Values.s3_access_key }}", + # "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", + # {{- end }} + "-Dweb.submit.enable=false", + "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", + "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager", + "-Dmetrics.reporter.prom.port=9251-9260", + "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager", + "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}"] ports: - containerPort: {{ .Values.taskmanager.rpc_port }} name: rpc From aaa4d9c4f947e51b89208609b3c365c99a579339 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 15:55:28 +0530 Subject: [PATCH 41/46] #OB-553 fix: Deployment issue fixes for OCI --- .../templates/flink_job_deployment.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index b85c7dd9ec..90aabc75eb 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -114,10 +114,10 @@ spec: "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}", {{- end }} {{- if eq .Values.checkpoint_store_type "s3" }} - "-Ds3.access-key={{ .Values.s3_access_key }}", - "-Ds3.secret-key={{ .Values.s3_secret_key }}", - "-Ds3.endpoint={{ .Values.s3_endpoint }}", - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", + "-Dpresto.s3.access-key={{ .Values.s3_access_key }}", + "-Dpresto.s3.secret-key={{ .Values.s3_secret_key }}", + "-Dpresto.s3.endpoint={{ .Values.s3_endpoint }}", + "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}", {{- end }} # {{- if eq .Values.checkpoint_store_type "gcloud" }} # "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}", @@ -200,10 +200,10 @@ spec: "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}", {{- end }} {{- if eq .Values.checkpoint_store_type "s3" }} - "-Ds3.access-key={{ .Values.s3_access_key }}", - "-Ds3.secret-key={{ .Values.s3_secret_key }}", - "-Ds3.endpoint={{ .Values.s3_endpoint }}", - "-Ds3.path.style.access={{ .Values.s3_path_style_access }}", + "-Dpresto.s3.access-key={{ .Values.s3_access_key }}", + "-Dpresto.s3.secret-key={{ .Values.s3_secret_key }}", + "-Dpresto.s3.endpoint={{ .Values.s3_endpoint }}", + "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}", {{- end }} # {{- if eq .Values.checkpoint_store_type "gcloud" }} # "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}", From c0b1257eb50a195fb5187c67a1150721b6855fcf Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Thu, 14 Dec 2023 17:24:46 +0530 Subject: [PATCH 42/46] #OB-553 fix: Deployment issue fixes for OCI --- data-pipeline-flink/sunbird-dp-distribution/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/data-pipeline-flink/sunbird-dp-distribution/Dockerfile b/data-pipeline-flink/sunbird-dp-distribution/Dockerfile index c220a7daa1..a1b0175ca9 100644 --- a/data-pipeline-flink/sunbird-dp-distribution/Dockerfile +++ b/data-pipeline-flink/sunbird-dp-distribution/Dockerfile @@ -4,6 +4,7 @@ USER flink RUN tar -xvf /tmp/sunbird-dp-distribution-1.0.tar.gz -C $FLINK_HOME/lib/ RUN mkdir $FLINK_HOME/plugins/s3-fs-presto RUN cp $FLINK_HOME/opt/flink-s3-fs-presto-1.13.5.jar $FLINK_HOME/plugins/s3-fs-presto/ +RUN cp $FLINK_HOME/opt/flink-s3-fs-presto-1.13.5.jar $FLINK_HOME/lib/flink-aaa-s3-fs-presto-1.13.5.jar USER root RUN rm -f /tmp/sunbird-dp-distribution-1.0.tar.gz USER flink From 7816ef76da99c71df4eafd0d5db8a0e511dda5d2 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Fri, 15 Dec 2023 13:36:47 +0530 Subject: [PATCH 43/46] #OB-553 fix: Deployment issue fixes for OCI --- kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml index 1944419946..c63417ebf0 100644 --- a/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml +++ b/kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml @@ -33,6 +33,9 @@ postgres_max_connections: 2 azure_account: "{{ cloud_private_storage_accountname }}" azure_secret: "{{ cloud_private_storage_secret }}" cloud_storage_flink_bucketname: dev-data-store + +s3_path_style_access: true + flink_dp_storage_container: "" checkpoint_store_type: "{{ druid_storage_type }}" checkpoint_interval: 60000 From c7485ee588fbbde921ca18a3e2c19b0634420579 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Fri, 15 Dec 2023 14:15:51 +0530 Subject: [PATCH 44/46] #OB-553 fix: Deployment issue fixes for OCI --- .../datapipeline_jobs/templates/flink_job_deployment.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index 90aabc75eb..cff7121a7b 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -117,7 +117,7 @@ spec: "-Dpresto.s3.access-key={{ .Values.s3_access_key }}", "-Dpresto.s3.secret-key={{ .Values.s3_secret_key }}", "-Dpresto.s3.endpoint={{ .Values.s3_endpoint }}", - "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}", + "-Dpresto.s3.path.style.access=true", {{- end }} # {{- if eq .Values.checkpoint_store_type "gcloud" }} # "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}", @@ -203,7 +203,7 @@ spec: "-Dpresto.s3.access-key={{ .Values.s3_access_key }}", "-Dpresto.s3.secret-key={{ .Values.s3_secret_key }}", "-Dpresto.s3.endpoint={{ .Values.s3_endpoint }}", - "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}", + "-Dpresto.s3.path.style.access=true", {{- end }} # {{- if eq .Values.checkpoint_store_type "gcloud" }} # "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}", From 01c76bb3d95c4f780bc8d36cccabab71bce0eaf1 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Fri, 15 Dec 2023 17:16:50 +0530 Subject: [PATCH 45/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/inventory/env/group_vars/all.yml | 2 +- .../templates/flink_job_deployment.yaml | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index 6e4f5cf267..d8d4464993 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -190,7 +190,7 @@ s3_storage_key: "{{ sunbird_private_s3_storage_key }}" s3_storage_secret: "{{ sunbird_private_s3_storage_secret }}" s3_storage_endpoint: "" s3_request_signature_version: AWS4-HMAC-SHA256 -s3_path_style_access: false +s3_path_style_access: true s3_https_only: false s3_default_bucket_location: "" s3_storage_container: "" diff --git a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml index cff7121a7b..57c420cdfa 100644 --- a/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml +++ b/kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml @@ -117,13 +117,13 @@ spec: "-Dpresto.s3.access-key={{ .Values.s3_access_key }}", "-Dpresto.s3.secret-key={{ .Values.s3_secret_key }}", "-Dpresto.s3.endpoint={{ .Values.s3_endpoint }}", - "-Dpresto.s3.path.style.access=true", + "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}", + {{- end }} + {{- if eq .Values.checkpoint_store_type "gcloud" }} + "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}", + "-Dfs.gs.auth.client.secret={{ .Values.s3_access_key }}", + "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", {{- end }} - # {{- if eq .Values.checkpoint_store_type "gcloud" }} - # "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}", - # "-Dfs.gs.auth.client.secret={{ .Values.s3_access_key }}", - # "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", - # {{- end }} "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}", @@ -203,13 +203,13 @@ spec: "-Dpresto.s3.access-key={{ .Values.s3_access_key }}", "-Dpresto.s3.secret-key={{ .Values.s3_secret_key }}", "-Dpresto.s3.endpoint={{ .Values.s3_endpoint }}", - "-Dpresto.s3.path.style.access=true", + "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}", + {{- end }} + {{- if eq .Values.checkpoint_store_type "gcloud" }} + "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}", + "-Dfs.gs.auth.client.secret={{ .Values.s3_access_key }}", + "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", {{- end }} - # {{- if eq .Values.checkpoint_store_type "gcloud" }} - # "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}", - # "-Dfs.gs.auth.client.secret={{ .Values.s3_access_key }}", - # "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}", - # {{- end }} "-Dweb.submit.enable=false", "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter", "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager", From 8a3ca375e139b3f5540b8da4ebe45b2c55cff398 Mon Sep 17 00:00:00 2001 From: sowmya-dixit Date: Mon, 18 Dec 2023 16:53:41 +0530 Subject: [PATCH 46/46] #OB-553 fix: Deployment issue fixes for OCI --- ansible/inventory/env/group_vars/all.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/inventory/env/group_vars/all.yml b/ansible/inventory/env/group_vars/all.yml index d8d4464993..0b59c0382d 100644 --- a/ansible/inventory/env/group_vars/all.yml +++ b/ansible/inventory/env/group_vars/all.yml @@ -171,9 +171,9 @@ druid_report_token: "Bearer {{ sunbird_api_auth_token }}" #redis multiprocess config content_port: 6379 -device_port: 6380 -user_port: 6381 -dialcode_port: 6382 +device_port: 6379 +user_port: 6379 +dialcode_port: 6379 ### kafka and zookeeper ip vars and being used in kafka topiccreation role. ingestion_zookeeper_ip: "{{ groups['ingestion-cluster-zookeeper'][0] }}"