diff --git a/ansible/roles/analytics-spark-provision/defaults/main.yml b/ansible/roles/analytics-spark-provision/defaults/main.yml index 4862fa39e5..f87de654de 100644 --- a/ansible/roles/analytics-spark-provision/defaults/main.yml +++ b/ansible/roles/analytics-spark-provision/defaults/main.yml @@ -3,7 +3,7 @@ analytics: soft_path: /mount/data/analytics base_path: /home/analytics scala_version: 2.12.10 -spark_version: 3.2.1 +spark_version: 3.1.3 model_version: "2.0" spark: home: "{{ analytics.home }}/spark-{{ spark_version }}-bin-hadoop2.7" diff --git a/ansible/roles/content-snapshot-indexer/defaults/main.yml b/ansible/roles/content-snapshot-indexer/defaults/main.yml index 436161e9c7..5b523f2c70 100644 --- a/ansible/roles/content-snapshot-indexer/defaults/main.yml +++ b/ansible/roles/content-snapshot-indexer/defaults/main.yml @@ -48,6 +48,7 @@ cloud_storage: provider: "azure" account_name: "{{cloud_public_storage_accountname}}" account_key: "{{cloud_public_storage_secret}}" + account_endpoint: "" cassandra: host: "{{lp_cassandra_host}}" ## LMS-Cassandra IP Address. diff --git a/kubernetes/helm_charts/secor/config/secor.azure.properties b/kubernetes/helm_charts/secor/config/secor.azure.properties index 23f8b76dc9..27c90d5e19 100644 --- a/kubernetes/helm_charts/secor/config/secor.azure.properties +++ b/kubernetes/helm_charts/secor/config/secor.azure.properties @@ -22,6 +22,20 @@ secor.azure.container.name={{ $.Values.azure_container_name }} # Microsoft Azure blob storage path where files are stored within the container. secor.azure.path={{- get (get $.Values.secor_jobs $.Release.Name) "base_path" }} +# Output file pattern excluding prefix. Defaults to topic/partition/generation_kafkaPartition_fmOffset.gz. +# Available placeholders are +# topic - The topic name the data is being fetched +# partition - The partition name +# generation - Generation +# kafkaPartition - The kafka partition +# fmOffset - First Message offset in the file. +# randomHex - A 4 character random hex to append to the file name +# currentTimestamp - Time of upload in epoch format +# currentTime - Time of upload in HH-mm format +# currentDate - Time of upload in YYYYMMDD format +# folder - Folder to use based on message id map lookup +secor.s3.output_file_pattern={{ get (get $.Values.secor_jobs $.Release.Name) "output_file_pattern" }} + ################ # END MUST SET # ################ \ No newline at end of file