#OB-553 fix: Deployment issue fixes for OCI

ansible/inventory/env/group_vars/all.yml

@@ -171,9 +171,9 @@ druid_report_token: "Bearer {{ sunbird_api_auth_token }}"
 
 #redis multiprocess config
 content_port: 6379
-device_port: 6380
-user_port: 6381
-dialcode_port: 6382
+device_port: 6379
+user_port: 6379
+dialcode_port: 6379
 
 ### kafka and zookeeper ip vars and being used in kafka topiccreation role.
 ingestion_zookeeper_ip: "{{ groups['ingestion-cluster-zookeeper'][0] }}"
@@ -190,7 +190,7 @@ s3_storage_key: "{{ sunbird_private_s3_storage_key }}"
 s3_storage_secret: "{{ sunbird_private_s3_storage_secret }}"
 s3_storage_endpoint: ""
 s3_request_signature_version: AWS4-HMAC-SHA256
-s3_path_style_access: false
+s3_path_style_access: true
 s3_https_only: false
 s3_default_bucket_location: ""
 s3_storage_container: ""

ansible/kubernetes/helm_charts/datapipeline/flink-jobs/values.j2

@@ -6,7 +6,7 @@ dockerhub: {{ dockerhub }}
 repository: {{ datapipeline_repository|default('data-pipeline') }}
 image_tag: {{ image_tag }}
 
-checkpoint_store_type: {{ cloud_service_provider }} 
+checkpoint_store_type: {{ druid_storage_type }} 
 cloud_storage_key: {{ cloud_private_storage_accountname }}
 cloud_storage_secret: {{ cloud_private_storage_secret }}
 cloud_storage_endpoint: {{ cloud_private_storage_endpoint }}

data-pipeline-flink/sunbird-dp-distribution/Dockerfile

@@ -4,6 +4,7 @@ USER flink
 RUN tar -xvf /tmp/sunbird-dp-distribution-1.0.tar.gz -C $FLINK_HOME/lib/
 RUN mkdir $FLINK_HOME/plugins/s3-fs-presto
 RUN cp $FLINK_HOME/opt/flink-s3-fs-presto-1.13.5.jar $FLINK_HOME/plugins/s3-fs-presto/
+RUN cp $FLINK_HOME/opt/flink-s3-fs-presto-1.13.5.jar $FLINK_HOME/lib/flink-aaa-s3-fs-presto-1.13.5.jar
 USER root
 RUN rm -f /tmp/sunbird-dp-distribution-1.0.tar.gz
 USER flink

kubernetes/ansible/roles/flink-jobs-deploy/defaults/main.yml

@@ -33,8 +33,11 @@ postgres_max_connections: 2
 azure_account: "{{ cloud_private_storage_accountname }}"
 azure_secret: "{{ cloud_private_storage_secret }}"
 cloud_storage_flink_bucketname: dev-data-store
+
+s3_path_style_access: true
+
 flink_dp_storage_container: ""
-checkpoint_store_type: gcloud
+checkpoint_store_type: "{{ druid_storage_type }}"
 checkpoint_interval: 60000
 checkpoint_pause_between_seconds: 5000
 checkpoint_compression_enabled: true

kubernetes/helm_charts/datapipeline_jobs/templates/flink_job_deployment.yaml

@@ -110,20 +110,20 @@ spec:
         command: ["/opt/flink/bin/standalone-job.sh"]
         args: ["start-foreground",
                "--job-classname={{ .Values.job_classname }}",
-{{- if eq .Values.checkpoint_store_type "azure" }}
-               "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net={{ .Values.cloud_storage_secret }}",
-{{- end }}
-{{- if eq .Values.checkpoint_store_type "s3" }}
-               "-Ds3.access-key={{ .Values.cloud_storage_key }}",
-               "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
-               "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}",
-               "-Ds3.path.style.access={{ .Values.s3_path_style_access }}",
-{{- end }}
-{{- if eq .Values.checkpoint_store_type "gcloud" }}
-               "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
-               "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
+              {{- if eq .Values.checkpoint_store_type "azure" }}
+               "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}",
+              {{- end }}
+              {{- if eq .Values.checkpoint_store_type "s3" }}
+               "-Dpresto.s3.access-key={{ .Values.s3_access_key }}",
+               "-Dpresto.s3.secret-key={{ .Values.s3_secret_key }}",
+               "-Dpresto.s3.endpoint={{ .Values.s3_endpoint }}",
+               "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}",
+              {{- end }}
+              {{- if eq .Values.checkpoint_store_type "gcloud" }}
+               "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}",
+               "-Dfs.gs.auth.client.secret={{ .Values.s3_access_key }}",
                "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}",
-{{- end }}
+              {{- end }}
                "-Dweb.submit.enable=false",
                "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter",
                "-Dmetrics.reporter.prom.port={{ .Values.jobmanager.prom_port }}",
@@ -196,26 +196,26 @@ spec:
         workingDir: {{ .Values.taskmanager.flink_work_dir }}
         command: ["/opt/flink/bin/taskmanager.sh"]
         args: ["start-foreground",
-{{- if eq .Values.checkpoint_store_type "azure" }}
-          "-Dfs.azure.account.key.{{ .Values.cloud_storage_key }}.blob.core.windows.net={{ .Values.cloud_storage_secret }}",
-{{- end }}
-{{- if eq .Values.checkpoint_store_type "s3" }}
-          "-Ds3.access-key={{ .Values.cloud_storage_key }}",
-          "-Ds3.secret-key={{ .Values.cloud_storage_secret }}",
-          "-Ds3.endpoint={{ .Values.cloud_storage_endpoint }}",
-          "-Ds3.path.style.access={{ .Values.s3_path_style_access }}",
-{{- end }}
-{{- if eq .Values.checkpoint_store_type "gcloud" }}
-          "-Dfs.gs.auth.client.id={{ .Values.cloud_storage_key }}",
-          "-Dfs.gs.auth.client.secret={{ .Values.cloud_storage_secret }}",
-          "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}",
-{{- end }}
-          "-Dweb.submit.enable=false",
-          "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter",
-          "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager",
-          "-Dmetrics.reporter.prom.port=9251-9260",
-          "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager",
-          "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}"]
+              {{- if eq .Values.checkpoint_store_type "azure" }}
+               "-Dfs.azure.account.key.{{ .Values.azure_account }}.blob.core.windows.net={{ .Values.azure_secret }}",
+              {{- end }}
+              {{- if eq .Values.checkpoint_store_type "s3" }}
+               "-Dpresto.s3.access-key={{ .Values.s3_access_key }}",
+               "-Dpresto.s3.secret-key={{ .Values.s3_secret_key }}",
+               "-Dpresto.s3.endpoint={{ .Values.s3_endpoint }}",
+               "-Dpresto.s3.path.style.access={{ .Values.s3_path_style_access }}",
+              {{- end }}
+              {{- if eq .Values.checkpoint_store_type "gcloud" }}
+               "-Dfs.gs.auth.client.id={{ .Values.s3_access_key }}",
+               "-Dfs.gs.auth.client.secret={{ .Values.s3_access_key }}",
+               "-Dfs.gs.project.id={{ .Values.cloud_storage_project_id }}",
+              {{- end }}
+               "-Dweb.submit.enable=false",
+               "-Dmetrics.reporter.prom.class=org.apache.flink.metrics.prometheus.PrometheusReporter",
+               "-Dmetrics.reporter.prom.host={{ .Release.Name }}-taskmanager",
+               "-Dmetrics.reporter.prom.port=9251-9260",
+               "-Djobmanager.rpc.address={{ .Release.Name }}-jobmanager",
+               "-Dtaskmanager.rpc.port={{ .Values.taskmanager.rpc_port }}"]
         ports:
         - containerPort: {{ .Values.taskmanager.rpc_port }}
           name: rpc

kubernetes/helm_charts/datapipeline_jobs/values.j2

@@ -3,7 +3,7 @@ imagepullsecrets: {{ imagepullsecrets }}
 dockerhub: {{ dockerhub }}
 repository: {{flink_repository|default('sunbird-datapipeline')}}
 image_tag: {{ image_tag }}
-checkpoint_store_type: {{ checkpoint_store_type }}
+checkpoint_store_type: {{ druid_storage_type }}
 azure_account: {{ azure_account }}
 azure_secret: {{ azure_secret }}
 s3_access_key: {{ cloud_private_storage_accountname }}
@@ -154,15 +154,15 @@ base_config: |
       statebackend {
         blob {
           storage {
-            account = "{% if checkpoint_store_type == "azure" %}{{ azure_account }}.blob.core.windows.net{% elif checkpoint_store_type == "s3" %}{{ flink_dp_storage_container }}{% elif checkpoint_store_type == "gcloud" %}{{ flink_dp_storage_container }}{% endif %}"
+            account = "{% if checkpoint_store_type == "azure" %}{{ azure_account }}.blob.core.windows.net{% elif checkpoint_store_type == "s3" %}{{ cloud_private_storage_accountname }}{% elif checkpoint_store_type == "gcloud" %}{{ flink_dp_storage_container }}{% endif %}"
             container = "{{ cloud_storage_flink_bucketname }}"
             checkpointing.dir = "checkpoint"
           }
         }
 {% if checkpoint_store_type == "azure" %}
         base.url = "wasbs://"${job.statebackend.blob.storage.container}"@"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.checkpointing.dir}
 {% elif checkpoint_store_type == "s3" %}
-        base.url = "s3://"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir}
+        base.url = "s3://"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir}
 {% elif checkpoint_store_type == "gcloud" %}
         base.url = "gs://"${job.statebackend.blob.storage.account}"/"${job.statebackend.blob.storage.container}"/"${job.statebackend.blob.storage.checkpointing.dir}
 {% endif %}