From 1c5053d519b4968e8c3ca93443f1ce41ea39f8c2 Mon Sep 17 00:00:00 2001 From: Lucas Date: Thu, 22 Aug 2024 11:44:19 +0200 Subject: [PATCH] fluent-bit 3.1.2 (#142) Features: * Upgrade to fluent-bit `3.1.2` for all distros * Enable amzn2023 arm64 Minor fixes: * Fix windows fluentbit download url * Sync centos7 mirrors * Wait before provisioning (non-trivial user_data script takes a while) * Use maintained fork (with updated crowdstrike hashes) --- .../playbook-provision-prerelease.yml | 42 ++++++++++++------- .../playbook-provision-repo.yml | 42 ++++++++++++------- .../requirements.yml | 2 +- .../user_data_script_for_ssm.tftpl | 5 +++ versions/amazonlinux_2023.yml | 6 +-- versions/common.yml | 2 +- versions/strategyMatrix.py | 2 +- 7 files changed, 62 insertions(+), 39 deletions(-) diff --git a/ansible/provision-and-execute-tests/playbook-provision-prerelease.yml b/ansible/provision-and-execute-tests/playbook-provision-prerelease.yml index a6ddbc96..a432550f 100644 --- a/ansible/provision-and-execute-tests/playbook-provision-prerelease.yml +++ b/ansible/provision-and-execute-tests/playbook-provision-prerelease.yml @@ -1,22 +1,32 @@ - name: (linux) (prerelease) Install CrowdStrike Falcon and NRIA from prod bucket hosts: linux become: true - gather_facts: yes - vars: - # install_crowdstrike_falcon role vars - falcon_client_id: "{{ lookup('env', 'CROWDSTRIKE_CLIENT_ID') }}" - falcon_client_secret: "{{ lookup('env', 'CROWDSTRIKE_CLIENT_SECRET') }}" - falcon_customer_id: "{{ lookup('env', 'CROWDSTRIKE_CUSTOMER_ID') }}" - api_base_url: "https://api.laggar.gcw.crowdstrike.com" - # newrelic_cli role vars - repo_endpoint: "https://nr-downloads-main.s3.amazonaws.com/" - recipe: "newrelic-infra" - region: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_REGION') }}" # US | EU | STAGING - nr_api_key: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_API_KEY') }}" - nr_account_id: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_ACCOUNT_ID') }}" - roles: - - caos.ansible_roles.install_crowdstrike_falcon - - caos.ansible_roles.newrelic_cli + gather_facts: no + tasks: + - name: Wait for connection to be available + wait_for_connection: + + - name: Gather facts for first time + ansible.builtin.setup: + + - name: Install crowdstrike falcon + ansible.builtin.include_role: + name: caos.ansible_roles.install_crowdstrike_falcon + vars: + falcon_client_id: "{{ lookup('env', 'CROWDSTRIKE_CLIENT_ID') }}" + falcon_client_secret: "{{ lookup('env', 'CROWDSTRIKE_CLIENT_SECRET') }}" + falcon_customer_id: "{{ lookup('env', 'CROWDSTRIKE_CUSTOMER_ID') }}" + api_base_url: "https://api.laggar.gcw.crowdstrike.com" + + - name: Install newrelic-infra agent + ansible.builtin.include_role: + name: caos.ansible_roles.newrelic_cli + vars: + repo_endpoint: "https://nr-downloads-main.s3.amazonaws.com/" + recipe: "newrelic-infra" + region: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_REGION') }}" # US | EU | STAGING + nr_api_key: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_API_KEY') }}" + nr_account_id: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_ACCOUNT_ID') }}" - name: Install fluent-bit from GH prerelease hosts: linux diff --git a/ansible/provision-and-execute-tests/playbook-provision-repo.yml b/ansible/provision-and-execute-tests/playbook-provision-repo.yml index 5f249317..a23986c1 100644 --- a/ansible/provision-and-execute-tests/playbook-provision-repo.yml +++ b/ansible/provision-and-execute-tests/playbook-provision-repo.yml @@ -1,19 +1,29 @@ - name: (linux) (production/staging) Install CrowdStrike Falcon and NRIA hosts: linux become: true - gather_facts: yes - vars: - # install_crowdstrike_falcon role vars - falcon_client_id: "{{ lookup('env', 'CROWDSTRIKE_CLIENT_ID') }}" - falcon_client_secret: "{{ lookup('env', 'CROWDSTRIKE_CLIENT_SECRET') }}" - falcon_customer_id: "{{ lookup('env', 'CROWDSTRIKE_CUSTOMER_ID') }}" - api_base_url: "https://api.laggar.gcw.crowdstrike.com" - # newrelic_cli role vars - repo_endpoint: "{{ lookup('ansible.builtin.env', 'NRIA_REPO_ENDPOINT') }}" - recipe: "newrelic-infra" - region: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_REGION') }}" # US | EU | STAGING - nr_api_key: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_API_KEY') }}" - nr_account_id: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_ACCOUNT_ID') }}" - roles: - - caos.ansible_roles.install_crowdstrike_falcon - - caos.ansible_roles.newrelic_cli + gather_facts: no + tasks: + - name: Wait for connection to be available + wait_for_connection: + + - name: Gather facts for first time + ansible.builtin.setup: + + - name: Install crowdstrike falcon + ansible.builtin.include_role: + name: caos.ansible_roles.install_crowdstrike_falcon + vars: + falcon_client_id: "{{ lookup('env', 'CROWDSTRIKE_CLIENT_ID') }}" + falcon_client_secret: "{{ lookup('env', 'CROWDSTRIKE_CLIENT_SECRET') }}" + falcon_customer_id: "{{ lookup('env', 'CROWDSTRIKE_CUSTOMER_ID') }}" + api_base_url: "https://api.laggar.gcw.crowdstrike.com" + + - name: Install newrelic-infra agent + ansible.builtin.include_role: + name: caos.ansible_roles.newrelic_cli + vars: + repo_endpoint: "{{ lookup('ansible.builtin.env', 'NRIA_REPO_ENDPOINT') }}" + recipe: "newrelic-infra" + region: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_REGION') }}" # US | EU | STAGING + nr_api_key: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_API_KEY') }}" + nr_account_id: "{{ lookup('ansible.builtin.env', 'NEW_RELIC_ACCOUNT_ID') }}" diff --git a/ansible/provision-and-execute-tests/requirements.yml b/ansible/provision-and-execute-tests/requirements.yml index 17807a4b..054380b1 100644 --- a/ansible/provision-and-execute-tests/requirements.yml +++ b/ansible/provision-and-execute-tests/requirements.yml @@ -6,7 +6,7 @@ collections: version: 2.11.0 - name: chocolatey.chocolatey version: 1.5.1 - - name: git+https://github.com/Sivakumar3695/caos-ansible-roles#/caos.ansible_roles/ + - name: git+https://github.com/luckslovez/caos-ansible-roles#/caos.ansible_roles/ type: git roles: diff --git a/terraform/ec2-instances-creator/user_data_script_for_ssm.tftpl b/terraform/ec2-instances-creator/user_data_script_for_ssm.tftpl index 88629f7f..20e6fdfd 100644 --- a/terraform/ec2-instances-creator/user_data_script_for_ssm.tftpl +++ b/terraform/ec2-instances-creator/user_data_script_for_ssm.tftpl @@ -1,6 +1,11 @@ #!/bin/bash %{if os_distro == "centos"} +%{if os_version == 7} +sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* +sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* +sudo yum distro-sync -y +%{endif} %{if os_version == 8} sudo sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-* sudo sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-* diff --git a/versions/amazonlinux_2023.yml b/versions/amazonlinux_2023.yml index 19b54036..a3ccad56 100644 --- a/versions/amazonlinux_2023.yml +++ b/versions/amazonlinux_2023.yml @@ -3,7 +3,5 @@ osVersion: 2023 packages: - arch: x86_64 ami: ami-01103fb68b3569475 -# Infra-agent needs to first enable al2023_arm64's use of fluent-bit -# https://github.com/newrelic/infrastructure-agent/blob/a9e09271ae4c7ebd30271591a8f597e21fca151a/build/goreleaser/linux/al2023_arm64.yml#L76 -# - arch: aarch64 -# ami: ami-01416d11f88e7cb55 \ No newline at end of file + - arch: aarch64 + ami: ami-0b9df99d3514cdede diff --git a/versions/common.yml b/versions/common.yml index eb42150e..7ea346f0 100644 --- a/versions/common.yml +++ b/versions/common.yml @@ -1,4 +1,4 @@ -fbVersion: 3.0.6 +fbVersion: 3.1.2 # This file, together with each distro file are processed and merged incrementally to # build all the information required to download and test each package. Each package ends diff --git a/versions/strategyMatrix.py b/versions/strategyMatrix.py index 72290514..4e4f47e8 100644 --- a/versions/strategyMatrix.py +++ b/versions/strategyMatrix.py @@ -138,7 +138,7 @@ def windows_package_details(data): windows_target_arch = {"win32": "386", "win64": "amd64"}[data["arch"]] target_package_name = f"fb-windows-{data['fbVersion']}-{windows_target_arch}.zip" return { - "packageUrl": f"http://fluentbit.io/releases/{get_major_minor_version(data['fbVersion'])}/fluent-bit-{data['fbVersion']}-{data['arch']}.zip", + "packageUrl": f"http://packages.fluentbit.io/windows/fluent-bit-{data['fbVersion']}-{data['arch']}.zip", "targetPackageName": target_package_name, "nrPackageUrl": f"https://logging-fb-windows-packages.s3.us-east-2.amazonaws.com/{target_package_name}", "isStaging": True, # No staging for windows