From 8988e98a140eda8eeffcda6f206afe0d5049261b Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Thu, 3 Mar 2022 17:47:17 +0000 Subject: [PATCH 01/11] chore: commit add-logging-agent-repo.sh script to repo --- .github/workflows/main.yaml | 33 +++ gce/add-logging-agent-repo.sh | 522 ++++++++++++++++++++++++++++++++++ gce/startup-script.sh | 13 +- 3 files changed, 561 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/main.yaml create mode 100644 gce/add-logging-agent-repo.sh diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml new file mode 100644 index 00000000..1f7cbf71 --- /dev/null +++ b/.github/workflows/main.yaml @@ -0,0 +1,33 @@ +# Copyright 2022 Google LLC + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# https://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: A workflow for updating the add-logging-agent-repo.sh script. +# Controls when the action will run. + +on: + schedule: + # * is a special character in YAML so you have to quote this string + # Run this Github Action every Tuesday at 7 AM UTC + - cron: '0 7 * * 2' + +jobs: + build: + name: Download latest add-logging-agent-repo.sh script. + runs-on: ubuntu-latest + # don't run the workflow on forks + if: ${{github.repository == 'googlecloudplatform/getting-started-python'}} + steps: + - name: Download script. + id: update-add-logging-agent-repo + run: curl -o gce/add-logging-agent-repo.sh -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh \ No newline at end of file diff --git a/gce/add-logging-agent-repo.sh b/gce/add-logging-agent-repo.sh new file mode 100644 index 00000000..e999d40d --- /dev/null +++ b/gce/add-logging-agent-repo.sh @@ -0,0 +1,522 @@ +#!/bin/bash +# Copyright 2020 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Add repository for the Google logging agent. +# +# This script adds the required apt or yum repository and installs or uninstalls +# the agent based on the corresponding flags. +# +# Available flags: +# * `--verbose`: +# Turns on verbose logging during the script execution, which is helpful for +# debugging purposes. +# +# * `--also-install`: +# Installs the agent after adding the agent package repository. If this flag +# is absent, the script only adds the agent package repository. This flag +# can not be run with the `--uninstall` flag. +# +# * `--version `: +# Sets the agent version for the script to install. Allowed formats: +# * `latest`: +# Adds an agent package repository that contains all agent versions, and +# installs the latest version of the agent. +# * `MAJOR_VERSION.*.*`: +# Adds an agent package repository that contains all agent versions up to +# this major version (e.g. `1.*.*`), and installs the latest version of +# the agent within the range of that major version. +# * `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION`: +# Adds an agent package repository that contains all agent versions, and +# installs the specified version of the agent (e.g. `3.2.1`). +# +# * `--uninstall`: +# Uninstalls the agent. This flag can not be run with the `--also-install` +# flag. +# +# * `--remove-repo`: +# Removes the corresponding agent package repository after installing or +# uninstalling the agent. +# +# * `--dry-run`: +# Triggers only a dry run of the script execution and prints out the +# commands that it is supposed to execute. This is helpful to know what +# actions the script will take. +# +# * `--structured`: +# When installing or uninstalling packages, use the structured catch-all +# config package. +# +# * `--unstructured`: +# When installing or uninstalling packages, use the unstructured catch-all +# config package. This is currently the default. +# +# Sample usage: +# * To add the repo that contains all agent versions, run: +# $ bash add-logging-agent-repo.sh +# +# * To add the repo and also install the agent, run: +# $ bash add-logging-agent-repo.sh --also-install --version= +# +# * To uninstall the agent run: +# $ bash add-logging-agent-repo.sh --uninstall +# +# * To uninstall the agent and remove the repo, run: +# $ bash add-logging-agent-repo.sh --uninstall --remove-repo +# +# * To run the script with verbose logging, run: +# $ bash add-logging-agent-repo.sh --also-install --verbose +# +# * To run the script in dry-run mode, run: +# $ bash add-logging-agent-repo.sh --also-install --dry-run +# +# The environment variable `DO_NOT_INSTALL_CATCH_ALL_CONFIG` can be set to +# prevent the google-fluentd-catch-all-config package from being installed. +# +# Internal usage only: +# The environment variable `REPO_SUFFIX` can be set to alter which repository is +# used. A dash (-) will be inserted prior to the supplied suffix. `REPO_SUFFIX` +# defaults to `all` which contains all agent versions across different major +# versions. The full repository name is: +# "google-cloud-logging-[-]-". + +# Ignore the return code of command substitution in variables. +# shellcheck disable=SC2155 +# +# Initialize var used to notify config management tools of when a change is made. +CHANGED=0 + +fail() { + echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" + exit 1 +} + +# Parsing flag value. +declare -a ACTIONS=() +DRY_RUN='' +VERBOSE='false' +declare -a STRUCTURED_FLAGS=() +while getopts -- '-:' OPTCHAR; do + case "${OPTCHAR}" in + -) + case "${OPTARG}" in + # Note: Do not remove entries from this list when deprecating flags. + # That would break user scripts that specify those flags. Instead, + # leave the flag in place but make it a noop. + also-install) ACTIONS+=('also-install') ;; + version=*) AGENT_VERSION="${OPTARG#*=}" ;; + uninstall) ACTIONS+=('uninstall') ;; + remove-repo) ACTIONS+=('remove-repo') ;; + dry-run) echo 'Starting dry run'; DRY_RUN='dryrun' ;; + verbose) VERBOSE='true' ;; + structured) STRUCTURED_FLAGS+=('true') ;; + unstructured) STRUCTURED_FLAGS+=('false') ;; + *) fail "Unknown option '${OPTARG}'." ;; + esac + esac +done +[[ "${ACTIONS[*]}" == *uninstall* || ( "${ACTIONS[*]}" == *remove-repo* && "${ACTIONS[*]}" != *also-install* )]] || \ + ACTIONS+=('add-repo') +# Sort the actions array for easier parsing. +readarray -t ACTIONS < <(printf '%s\n' "${ACTIONS[@]}" | sort) +readonly ACTIONS DRY_RUN VERBOSE +# Sort the structured flags array for easier parsing. +readarray -t STRUCTURED_FLAGS < <(printf '%s\n' "${STRUCTURED_FLAGS[@]}" | sort) +readonly STRUCTURED_FLAGS + +if [[ "${ACTIONS[*]}" == *also-install*uninstall* ]]; then + fail "Received conflicting flags 'also-install' and 'uninstall'." +fi + +if [[ "${STRUCTURED_FLAGS[*]}" == *false*true* ]]; then + fail "Received conflicting flags 'structured' and 'unstructured'." +fi + +if [[ ! ("${ACTIONS[*]}" == *also-install* || "${ACTIONS[*]}" == *uninstall* ) && -n "${STRUCTURED_FLAGS[*]}" ]]; then + fail "The 'structured' and 'unstructured' flags are only used in 'also-install' or 'uninstall' mode." +fi + +if [[ "${VERBOSE}" == 'true' ]]; then + echo 'Enable verbose logging.' + set -x +fi + +# Host that serves the repositories. +REPO_HOST='packages.cloud.google.com' + +# URL for the logging agent documentation. +AGENT_DOCS_URL='https://cloud.google.com/logging/docs/agent' + +# URL documentation which lists supported platforms for running the logging agent. +AGENT_SUPPORTED_URL="${AGENT_DOCS_URL}/#agent-os-list" + +# Packages to install. +AGENT_PACKAGE='google-fluentd' +if [[ -z "${DO_NOT_INSTALL_CATCH_ALL_CONFIG:-}" ]]; then + if [[ "${STRUCTURED_FLAGS[*]}" == *true* ]]; then + declare -a ADDITIONAL_PACKAGES=('google-fluentd-catch-all-config-structured') + else + declare -a ADDITIONAL_PACKAGES=('google-fluentd-catch-all-config') + fi +fi + +if [[ -f /etc/os-release ]]; then + . /etc/os-release +fi + +# If dry-run mode is enabled, echo VM state-changing commands instead of executing them. +dryrun() { + # Needed for commands that use pipes. + if [[ ! -t 0 ]]; then + cat + fi + printf -v cmd_str '%q ' "$@" + echo "DRY_RUN: Not executing '$cmd_str'" +} + +refresh_failed() { + local REPO_TYPE="$1" + local OS_FAMILY="$2" + fail "Could not refresh the google-cloud-logging ${REPO_TYPE} repositories. +Please check your network connectivity and make sure you are running a supported +${OS_FAMILY} distribution. See ${AGENT_SUPPORTED_URL} +for a list of supported platforms." +} + +resolve_version() { + if [[ "${AGENT_VERSION:-latest}" == 'latest' ]]; then + AGENT_VERSION='' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + REPO_SUFFIX="${REPO_SUFFIX:-"${AGENT_VERSION%%.*}"}" + elif ! grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + fail "The agent version [${AGENT_VERSION}] is not allowed. Expected values: [latest], +or anything in the format of [MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION] or [MAJOR_VERSION.*.*]." + fi +} + +handle_debian() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-oDebug::pkgAcquire::Worker=1) + + add_repo() { + [[ -n "${REPO_CODENAME:-}" ]] || lsb_release -v >/dev/null 2>&1 || { \ + apt-get update; apt-get -y install lsb-release; CHANGED=1; + } + [[ "$(dpkg -l apt-transport-https 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install apt-transport-https; CHANGED=1; + } + [[ "$(dpkg -l ca-certificates 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install ca-certificates; CHANGED=1; + } + local CODENAME="${REPO_CODENAME:-"$(lsb_release -sc)"}" + local REPO_NAME="google-cloud-logging-${CODENAME}-${REPO_SUFFIX:-all}" + local REPO_DATA="deb https://${REPO_HOST}/apt ${REPO_NAME} main" + if ! cmp -s <<<"${REPO_DATA}" - /etc/apt/sources.list.d/google-cloud-logging.list; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/apt/sources.list.d/google-cloud-logging.list + ${DRY_RUN} curl --connect-timeout 5 -s -f "https://${REPO_HOST}/apt/doc/apt-key.gpg" \ + | ${DRY_RUN} apt-key add - + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/apt/sources.list.d/google-cloud-logging.list ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/apt/sources.list.d/google-cloud-logging.list + CHANGED=1 + fi + } + + expected_version_installed() { + [[ "$(dpkg -l "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" 2>&1 | grep -o '^[a-z][a-z]' | sort -u)" == 'ii' ]] || \ + return + if [[ -z "${AGENT_VERSION:-}" ]]; then + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" && \ + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + else + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" + fi + } + + install_agent() { + ${DRY_RUN} apt-get update || refresh_failed 'apt' "${ID}" + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # apt package version format: e.g. 1.8.0-1. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="=${AGENT_VERSION}-*" + else + AGENT_VERSION="=${AGENT_VERSION%.\*}" + fi + fi + ${DRY_RUN} apt-get -y --allow-downgrades "${EXTRA_OPTS[@]}" install "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall_agent() { + # Return early unless at least one package is installed. + dpkg -l "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" 2>&1 | grep -qo '^ii' || return + ${DRY_RUN} apt-get -y "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" || \ + fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_rpm() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-v) + + add_repo() { + local REPO_NAME="google-cloud-logging-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + local REPO_DATA="\ +[google-cloud-logging] +name=Google Cloud Logging Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/yum.repos.d/google-cloud-logging.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/yum.repos.d/google-cloud-logging.repo + # After repo upgrades, CentOS7/RHEL7 won't pick up newly available packages + # until the cache is cleared. + ${DRY_RUN} rm -rf /var/cache/yum/*/*/google-cloud-logging/ + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/yum.repos.d/google-cloud-logging.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/yum.repos.d/google-cloud-logging.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + grep -qE "${AGENT_VERSION}" <<<"${CURRENT_VERSION}" && \ + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + else + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + [[ "${AGENT_VERSION}" == "${CURRENT_VERSION}" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + ${DRY_RUN} yum -y list updates || refresh_failed 'yum' "${ID}" + local COMMAND='install' + if [[ -n "${AGENT_VERSION:-}" ]]; then + [[ -z "${CURRENT_VERSION:-}" ]] || \ + [[ "${AGENT_VERSION}" == "$(sort -rV <<<"${AGENT_VERSION}"$'\n'"${CURRENT_VERSION}" | head -1)" ]] || \ + COMMAND='downgrade' + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # yum package version format: e.g. 1.0.1-1.el8. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="-${AGENT_VERSION}-1*" + else + AGENT_VERSION="-${AGENT_VERSION}" + fi + fi + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" "${COMMAND}" "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall_agent() { + # Return early if none of the packages are installed. + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" || \ + fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_redhat() { + local MAJOR_VERSION="$(rpm --eval %{?rhel})" + CODENAME="el${MAJOR_VERSION}" + handle_rpm +} + +handle_amazon_linux() { + CODENAME='el6' + handle_rpm +} + +handle_suse() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-vv) + + add_repo() { + local SUSE_VERSION=${VERSION_ID%%.*} + local CODENAME="sles${SUSE_VERSION}" + local REPO_NAME="google-cloud-logging-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + { + ${DRY_RUN} zypper --non-interactive refresh || { \ + echo >&2 'Could not refresh zypper repositories.'; \ + echo >&2 'This is not necessarily a fatal error; proceeding...'; \ + } + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + local REPO_DATA="\ +[google-cloud-logging] +name=Google Cloud Logging Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/zypp/repos.d/google-cloud-logging.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/zypp/repos.d/google-cloud-logging.repo + CHANGED=1 + fi + local RPM_KEYS="$(rpm --query gpg-pubkey)" # Save the installed keys. + ${DRY_RUN} rpm --import "https://${REPO_HOST}/yum/doc/yum-key.gpg" "https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if [[ -n "${DRY_RUN:-}" ]] || ! cmp --silent <<<"${RPM_KEYS}" - <(rpm --query gpg-pubkey); then + CHANGED=1 + fi + { + ${DRY_RUN} zypper --non-interactive --gpg-auto-import-keys refresh google-cloud-logging || \ + refresh_failed 'zypper' "${ID}"; \ + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + } + + remove_repo() { + if [[ -f /etc/zypp/repos.d/google-cloud-logging.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/zypp/repos.d/google-cloud-logging.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}" | grep -qE "${AGENT_VERSION}" && \ + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + else + [[ "${AGENT_VERSION}" == "$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # zypper package version format: e.g. 1.0.6-1.sles15. + if grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="<$(( ${AGENT_VERSION%%.*} + 1 ))" + else + AGENT_VERSION="=${AGENT_VERSION}" + fi + fi + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" install --oldpackage "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall_agent() { + # Return early if none of the packages are installed. + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" || \ + fail "${AGENT_PACKAGE} uninstallation failed." + # zypper doesn't like removing packages that are not installed. + if rpm -q "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$'; then + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${ADDITIONAL_PACKAGES[@]}" || \ + fail "${ADDITIONAL_PACKAGES[*]} uninstallation failed." + fi + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." + CHANGED=1 + } +} + +main() { + case "${ID:-}" in + amzn) handle_amazon_linux ;; + debian|ubuntu) handle_debian ;; + rhel|centos) handle_redhat ;; + sles|opensuse-leap) handle_suse ;; + *) + # Fallback for systems lacking /etc/os-release. + if [[ -f /etc/debian_version ]]; then + ID='debian' + handle_debian + elif [[ -f /etc/redhat-release ]]; then + ID='rhel' + handle_redhat + elif [[ -f /etc/SuSE-release ]]; then + ID='sles' + handle_suse + else + fail "Unidentifiable or unsupported platform. See +${AGENT_SUPPORTED_URL} for a list of supported platforms." + fi + esac + + if [[ "${ACTIONS[*]}" == *add-repo* ]]; then + resolve_version + add_repo + fi + if [[ "${ACTIONS[*]}" == *also-install* ]]; then + install_agent + elif [[ "${ACTIONS[*]}" == *uninstall* ]]; then + uninstall_agent + fi + if [[ "${ACTIONS[*]}" == *remove-repo* ]]; then + remove_repo + fi + + if [[ "${CHANGED}" == 0 ]]; then + echo 'No changes made.' + fi + + if [[ -n "${DRY_RUN:-}" ]]; then + echo 'Finished dry run. This was only a simulation, remove the --dry-run flag +to perform an actual execution of the script.' + fi +} + +main "$@" diff --git a/gce/startup-script.sh b/gce/startup-script.sh index 2c1712bc..d18e0c54 100644 --- a/gce/startup-script.sh +++ b/gce/startup-script.sh @@ -16,9 +16,12 @@ set -v # [START getting_started_gce_startup_script] -# Install Stackdriver logging agent -curl -sSO https://dl.google.com/cloudagents/install-logging-agent.sh -sudo bash install-logging-agent.sh +# Fetch source code +export HOME=/root +git clone https://github.com/GoogleCloudPlatform/getting-started-python.git /opt/app + +# Install Cloud Logging agent +sudo bash /opt/app/gce/add-logging-agent-repo.sh --also-install # Install or update needed software apt-get update @@ -28,10 +31,6 @@ pip install --upgrade pip virtualenv # Account to own server process useradd -m -d /home/pythonapp pythonapp -# Fetch source code -export HOME=/root -git clone https://github.com/GoogleCloudPlatform/getting-started-python.git /opt/app - # Python environment setup virtualenv -p python3 /opt/app/gce/env /bin/bash -c "source /opt/app/gce/env/bin/activate" From 1b39f19f3ab0f2aa5418a174e6a82810c64986e1 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Thu, 3 Mar 2022 17:59:32 +0000 Subject: [PATCH 02/11] chore: use workflow dispatch for testing --- .github/workflows/main.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 1f7cbf71..4fc95a4c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -15,11 +15,11 @@ name: A workflow for updating the add-logging-agent-repo.sh script. # Controls when the action will run. -on: - schedule: - # * is a special character in YAML so you have to quote this string - # Run this Github Action every Tuesday at 7 AM UTC - - cron: '0 7 * * 2' +on: workflow_dispatch + # schedule: + # # * is a special character in YAML so you have to quote this string + # # Run this Github Action every Tuesday at 7 AM UTC + # - cron: '0 7 * * 2' jobs: build: From 363fc1f8cdb0c68810b560821d5e0a9135ae8d52 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Thu, 3 Mar 2022 18:00:39 +0000 Subject: [PATCH 03/11] chore: run on forks too --- .github/workflows/main.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 4fc95a4c..d56eb63e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -25,8 +25,6 @@ jobs: build: name: Download latest add-logging-agent-repo.sh script. runs-on: ubuntu-latest - # don't run the workflow on forks - if: ${{github.repository == 'googlecloudplatform/getting-started-python'}} steps: - name: Download script. id: update-add-logging-agent-repo From 873ec9a49481856282149b40abbfe170b546ddbf Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Thu, 3 Mar 2022 21:55:58 +0000 Subject: [PATCH 04/11] chore: specify working-directory --- .github/workflows/main.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d56eb63e..2b29fdbe 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -28,4 +28,5 @@ jobs: steps: - name: Download script. id: update-add-logging-agent-repo - run: curl -o gce/add-logging-agent-repo.sh -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh \ No newline at end of file + run: curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh + working-directory: ./gce \ No newline at end of file From 646521b4682aeda9b2d826c7032855d98545f68d Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Thu, 3 Mar 2022 22:02:53 +0000 Subject: [PATCH 05/11] chore: try again --- .github/workflows/main.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 2b29fdbe..d56eb63e 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -28,5 +28,4 @@ jobs: steps: - name: Download script. id: update-add-logging-agent-repo - run: curl -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh - working-directory: ./gce \ No newline at end of file + run: curl -o gce/add-logging-agent-repo.sh -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh \ No newline at end of file From 1ee2859b31116447bdcacbbf47da2683993823c2 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 1 Apr 2022 15:02:33 +0000 Subject: [PATCH 06/11] chore: try wget --- .github/workflows/main.yaml | 3 +- gce/add-logging-agent-repo.sh | 522 ---------------------------------- 2 files changed, 2 insertions(+), 523 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d56eb63e..06d5e125 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -28,4 +28,5 @@ jobs: steps: - name: Download script. id: update-add-logging-agent-repo - run: curl -o gce/add-logging-agent-repo.sh -sSO https://dl.google.com/cloudagents/add-logging-agent-repo.sh \ No newline at end of file + run: wget -O ./gce/add-logging-agent-repo.sh https://dl.google.com/cloudagents/add-logging-agent-repo.sh + \ No newline at end of file diff --git a/gce/add-logging-agent-repo.sh b/gce/add-logging-agent-repo.sh index e999d40d..e69de29b 100644 --- a/gce/add-logging-agent-repo.sh +++ b/gce/add-logging-agent-repo.sh @@ -1,522 +0,0 @@ -#!/bin/bash -# Copyright 2020 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Add repository for the Google logging agent. -# -# This script adds the required apt or yum repository and installs or uninstalls -# the agent based on the corresponding flags. -# -# Available flags: -# * `--verbose`: -# Turns on verbose logging during the script execution, which is helpful for -# debugging purposes. -# -# * `--also-install`: -# Installs the agent after adding the agent package repository. If this flag -# is absent, the script only adds the agent package repository. This flag -# can not be run with the `--uninstall` flag. -# -# * `--version `: -# Sets the agent version for the script to install. Allowed formats: -# * `latest`: -# Adds an agent package repository that contains all agent versions, and -# installs the latest version of the agent. -# * `MAJOR_VERSION.*.*`: -# Adds an agent package repository that contains all agent versions up to -# this major version (e.g. `1.*.*`), and installs the latest version of -# the agent within the range of that major version. -# * `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION`: -# Adds an agent package repository that contains all agent versions, and -# installs the specified version of the agent (e.g. `3.2.1`). -# -# * `--uninstall`: -# Uninstalls the agent. This flag can not be run with the `--also-install` -# flag. -# -# * `--remove-repo`: -# Removes the corresponding agent package repository after installing or -# uninstalling the agent. -# -# * `--dry-run`: -# Triggers only a dry run of the script execution and prints out the -# commands that it is supposed to execute. This is helpful to know what -# actions the script will take. -# -# * `--structured`: -# When installing or uninstalling packages, use the structured catch-all -# config package. -# -# * `--unstructured`: -# When installing or uninstalling packages, use the unstructured catch-all -# config package. This is currently the default. -# -# Sample usage: -# * To add the repo that contains all agent versions, run: -# $ bash add-logging-agent-repo.sh -# -# * To add the repo and also install the agent, run: -# $ bash add-logging-agent-repo.sh --also-install --version= -# -# * To uninstall the agent run: -# $ bash add-logging-agent-repo.sh --uninstall -# -# * To uninstall the agent and remove the repo, run: -# $ bash add-logging-agent-repo.sh --uninstall --remove-repo -# -# * To run the script with verbose logging, run: -# $ bash add-logging-agent-repo.sh --also-install --verbose -# -# * To run the script in dry-run mode, run: -# $ bash add-logging-agent-repo.sh --also-install --dry-run -# -# The environment variable `DO_NOT_INSTALL_CATCH_ALL_CONFIG` can be set to -# prevent the google-fluentd-catch-all-config package from being installed. -# -# Internal usage only: -# The environment variable `REPO_SUFFIX` can be set to alter which repository is -# used. A dash (-) will be inserted prior to the supplied suffix. `REPO_SUFFIX` -# defaults to `all` which contains all agent versions across different major -# versions. The full repository name is: -# "google-cloud-logging-[-]-". - -# Ignore the return code of command substitution in variables. -# shellcheck disable=SC2155 -# -# Initialize var used to notify config management tools of when a change is made. -CHANGED=0 - -fail() { - echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" - exit 1 -} - -# Parsing flag value. -declare -a ACTIONS=() -DRY_RUN='' -VERBOSE='false' -declare -a STRUCTURED_FLAGS=() -while getopts -- '-:' OPTCHAR; do - case "${OPTCHAR}" in - -) - case "${OPTARG}" in - # Note: Do not remove entries from this list when deprecating flags. - # That would break user scripts that specify those flags. Instead, - # leave the flag in place but make it a noop. - also-install) ACTIONS+=('also-install') ;; - version=*) AGENT_VERSION="${OPTARG#*=}" ;; - uninstall) ACTIONS+=('uninstall') ;; - remove-repo) ACTIONS+=('remove-repo') ;; - dry-run) echo 'Starting dry run'; DRY_RUN='dryrun' ;; - verbose) VERBOSE='true' ;; - structured) STRUCTURED_FLAGS+=('true') ;; - unstructured) STRUCTURED_FLAGS+=('false') ;; - *) fail "Unknown option '${OPTARG}'." ;; - esac - esac -done -[[ "${ACTIONS[*]}" == *uninstall* || ( "${ACTIONS[*]}" == *remove-repo* && "${ACTIONS[*]}" != *also-install* )]] || \ - ACTIONS+=('add-repo') -# Sort the actions array for easier parsing. -readarray -t ACTIONS < <(printf '%s\n' "${ACTIONS[@]}" | sort) -readonly ACTIONS DRY_RUN VERBOSE -# Sort the structured flags array for easier parsing. -readarray -t STRUCTURED_FLAGS < <(printf '%s\n' "${STRUCTURED_FLAGS[@]}" | sort) -readonly STRUCTURED_FLAGS - -if [[ "${ACTIONS[*]}" == *also-install*uninstall* ]]; then - fail "Received conflicting flags 'also-install' and 'uninstall'." -fi - -if [[ "${STRUCTURED_FLAGS[*]}" == *false*true* ]]; then - fail "Received conflicting flags 'structured' and 'unstructured'." -fi - -if [[ ! ("${ACTIONS[*]}" == *also-install* || "${ACTIONS[*]}" == *uninstall* ) && -n "${STRUCTURED_FLAGS[*]}" ]]; then - fail "The 'structured' and 'unstructured' flags are only used in 'also-install' or 'uninstall' mode." -fi - -if [[ "${VERBOSE}" == 'true' ]]; then - echo 'Enable verbose logging.' - set -x -fi - -# Host that serves the repositories. -REPO_HOST='packages.cloud.google.com' - -# URL for the logging agent documentation. -AGENT_DOCS_URL='https://cloud.google.com/logging/docs/agent' - -# URL documentation which lists supported platforms for running the logging agent. -AGENT_SUPPORTED_URL="${AGENT_DOCS_URL}/#agent-os-list" - -# Packages to install. -AGENT_PACKAGE='google-fluentd' -if [[ -z "${DO_NOT_INSTALL_CATCH_ALL_CONFIG:-}" ]]; then - if [[ "${STRUCTURED_FLAGS[*]}" == *true* ]]; then - declare -a ADDITIONAL_PACKAGES=('google-fluentd-catch-all-config-structured') - else - declare -a ADDITIONAL_PACKAGES=('google-fluentd-catch-all-config') - fi -fi - -if [[ -f /etc/os-release ]]; then - . /etc/os-release -fi - -# If dry-run mode is enabled, echo VM state-changing commands instead of executing them. -dryrun() { - # Needed for commands that use pipes. - if [[ ! -t 0 ]]; then - cat - fi - printf -v cmd_str '%q ' "$@" - echo "DRY_RUN: Not executing '$cmd_str'" -} - -refresh_failed() { - local REPO_TYPE="$1" - local OS_FAMILY="$2" - fail "Could not refresh the google-cloud-logging ${REPO_TYPE} repositories. -Please check your network connectivity and make sure you are running a supported -${OS_FAMILY} distribution. See ${AGENT_SUPPORTED_URL} -for a list of supported platforms." -} - -resolve_version() { - if [[ "${AGENT_VERSION:-latest}" == 'latest' ]]; then - AGENT_VERSION='' - elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then - REPO_SUFFIX="${REPO_SUFFIX:-"${AGENT_VERSION%%.*}"}" - elif ! grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then - fail "The agent version [${AGENT_VERSION}] is not allowed. Expected values: [latest], -or anything in the format of [MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION] or [MAJOR_VERSION.*.*]." - fi -} - -handle_debian() { - declare -a EXTRA_OPTS=() - [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-oDebug::pkgAcquire::Worker=1) - - add_repo() { - [[ -n "${REPO_CODENAME:-}" ]] || lsb_release -v >/dev/null 2>&1 || { \ - apt-get update; apt-get -y install lsb-release; CHANGED=1; - } - [[ "$(dpkg -l apt-transport-https 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ - ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install apt-transport-https; CHANGED=1; - } - [[ "$(dpkg -l ca-certificates 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ - ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install ca-certificates; CHANGED=1; - } - local CODENAME="${REPO_CODENAME:-"$(lsb_release -sc)"}" - local REPO_NAME="google-cloud-logging-${CODENAME}-${REPO_SUFFIX:-all}" - local REPO_DATA="deb https://${REPO_HOST}/apt ${REPO_NAME} main" - if ! cmp -s <<<"${REPO_DATA}" - /etc/apt/sources.list.d/google-cloud-logging.list; then - echo "Adding agent repository for ${ID}." - ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/apt/sources.list.d/google-cloud-logging.list - ${DRY_RUN} curl --connect-timeout 5 -s -f "https://${REPO_HOST}/apt/doc/apt-key.gpg" \ - | ${DRY_RUN} apt-key add - - CHANGED=1 - fi - } - - remove_repo() { - if [[ -f /etc/apt/sources.list.d/google-cloud-logging.list ]]; then - echo "Removing agent repository for ${ID}." - ${DRY_RUN} rm /etc/apt/sources.list.d/google-cloud-logging.list - CHANGED=1 - fi - } - - expected_version_installed() { - [[ "$(dpkg -l "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" 2>&1 | grep -o '^[a-z][a-z]' | sort -u)" == 'ii' ]] || \ - return - if [[ -z "${AGENT_VERSION:-}" ]]; then - apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ - | grep -qo '^0 upgraded, 0 newly installed' - elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then - dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" && \ - apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ - | grep -qo '^0 upgraded, 0 newly installed' - else - dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" - fi - } - - install_agent() { - ${DRY_RUN} apt-get update || refresh_failed 'apt' "${ID}" - expected_version_installed || { \ - if [[ -n "${AGENT_VERSION:-}" ]]; then - # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. - # apt package version format: e.g. 1.8.0-1. - if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then - AGENT_VERSION="=${AGENT_VERSION}-*" - else - AGENT_VERSION="=${AGENT_VERSION%.\*}" - fi - fi - ${DRY_RUN} apt-get -y --allow-downgrades "${EXTRA_OPTS[@]}" install "${AGENT_PACKAGE}${AGENT_VERSION}" \ - "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ -installation failed." - echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." - CHANGED=1 - } - } - - uninstall_agent() { - # Return early unless at least one package is installed. - dpkg -l "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" 2>&1 | grep -qo '^ii' || return - ${DRY_RUN} apt-get -y "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" || \ - fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation failed." - echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." - CHANGED=1 - } -} - -handle_rpm() { - declare -a EXTRA_OPTS=() - [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-v) - - add_repo() { - local REPO_NAME="google-cloud-logging-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" - local REPO_DATA="\ -[google-cloud-logging] -name=Google Cloud Logging Agent Repository -baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} -autorefresh=0 -enabled=1 -type=rpm-md -gpgcheck=1 -repo_gpgcheck=0 -gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg - https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" - if ! cmp -s <<<"${REPO_DATA}" - /etc/yum.repos.d/google-cloud-logging.repo; then - echo "Adding agent repository for ${ID}." - ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/yum.repos.d/google-cloud-logging.repo - # After repo upgrades, CentOS7/RHEL7 won't pick up newly available packages - # until the cache is cleared. - ${DRY_RUN} rm -rf /var/cache/yum/*/*/google-cloud-logging/ - CHANGED=1 - fi - } - - remove_repo() { - if [[ -f /etc/yum.repos.d/google-cloud-logging.repo ]]; then - echo "Removing agent repository for ${ID}." - ${DRY_RUN} rm /etc/yum.repos.d/google-cloud-logging.repo - CHANGED=1 - fi - } - - expected_version_installed() { - rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return - if [[ -z "${AGENT_VERSION:-}" ]]; then - yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 - elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then - CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" - grep -qE "${AGENT_VERSION}" <<<"${CURRENT_VERSION}" && \ - yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 - else - CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" - [[ "${AGENT_VERSION}" == "${CURRENT_VERSION}" ]] - fi - } - - install_agent() { - expected_version_installed || { \ - ${DRY_RUN} yum -y list updates || refresh_failed 'yum' "${ID}" - local COMMAND='install' - if [[ -n "${AGENT_VERSION:-}" ]]; then - [[ -z "${CURRENT_VERSION:-}" ]] || \ - [[ "${AGENT_VERSION}" == "$(sort -rV <<<"${AGENT_VERSION}"$'\n'"${CURRENT_VERSION}" | head -1)" ]] || \ - COMMAND='downgrade' - # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. - # yum package version format: e.g. 1.0.1-1.el8. - if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then - AGENT_VERSION="-${AGENT_VERSION}-1*" - else - AGENT_VERSION="-${AGENT_VERSION}" - fi - fi - ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" "${COMMAND}" "${AGENT_PACKAGE}${AGENT_VERSION}" \ - "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ -installation failed." - echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." - CHANGED=1 - } - } - - uninstall_agent() { - # Return early if none of the packages are installed. - rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$' || return - ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" || \ - fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation failed." - echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." - CHANGED=1 - } -} - -handle_redhat() { - local MAJOR_VERSION="$(rpm --eval %{?rhel})" - CODENAME="el${MAJOR_VERSION}" - handle_rpm -} - -handle_amazon_linux() { - CODENAME='el6' - handle_rpm -} - -handle_suse() { - declare -a EXTRA_OPTS=() - [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-vv) - - add_repo() { - local SUSE_VERSION=${VERSION_ID%%.*} - local CODENAME="sles${SUSE_VERSION}" - local REPO_NAME="google-cloud-logging-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" - { - ${DRY_RUN} zypper --non-interactive refresh || { \ - echo >&2 'Could not refresh zypper repositories.'; \ - echo >&2 'This is not necessarily a fatal error; proceeding...'; \ - } - } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 - local REPO_DATA="\ -[google-cloud-logging] -name=Google Cloud Logging Agent Repository -baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} -autorefresh=0 -enabled=1 -type=rpm-md -gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg - https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" - if ! cmp -s <<<"${REPO_DATA}" - /etc/zypp/repos.d/google-cloud-logging.repo; then - echo "Adding agent repository for ${ID}." - ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/zypp/repos.d/google-cloud-logging.repo - CHANGED=1 - fi - local RPM_KEYS="$(rpm --query gpg-pubkey)" # Save the installed keys. - ${DRY_RUN} rpm --import "https://${REPO_HOST}/yum/doc/yum-key.gpg" "https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" - if [[ -n "${DRY_RUN:-}" ]] || ! cmp --silent <<<"${RPM_KEYS}" - <(rpm --query gpg-pubkey); then - CHANGED=1 - fi - { - ${DRY_RUN} zypper --non-interactive --gpg-auto-import-keys refresh google-cloud-logging || \ - refresh_failed 'zypper' "${ID}"; \ - } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 - } - - remove_repo() { - if [[ -f /etc/zypp/repos.d/google-cloud-logging.repo ]]; then - echo "Removing agent repository for ${ID}." - ${DRY_RUN} rm /etc/zypp/repos.d/google-cloud-logging.repo - CHANGED=1 - fi - } - - expected_version_installed() { - rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return - if [[ -z "${AGENT_VERSION:-}" ]]; then - zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ - | grep -qE '^Nothing to do.' - elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then - rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}" | grep -qE "${AGENT_VERSION}" && \ - zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ - | grep -qE '^Nothing to do.' - else - [[ "${AGENT_VERSION}" == "$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" ]] - fi - } - - install_agent() { - expected_version_installed || { \ - if [[ -n "${AGENT_VERSION:-}" ]]; then - # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. - # zypper package version format: e.g. 1.0.6-1.sles15. - if grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then - AGENT_VERSION="<$(( ${AGENT_VERSION%%.*} + 1 ))" - else - AGENT_VERSION="=${AGENT_VERSION}" - fi - fi - ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" install --oldpackage "${AGENT_PACKAGE}${AGENT_VERSION}" \ - "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ -installation failed." - echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." - CHANGED=1 - } - } - - uninstall_agent() { - # Return early if none of the packages are installed. - rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$' || return - ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" || \ - fail "${AGENT_PACKAGE} uninstallation failed." - # zypper doesn't like removing packages that are not installed. - if rpm -q "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$'; then - ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${ADDITIONAL_PACKAGES[@]}" || \ - fail "${ADDITIONAL_PACKAGES[*]} uninstallation failed." - fi - echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." - CHANGED=1 - } -} - -main() { - case "${ID:-}" in - amzn) handle_amazon_linux ;; - debian|ubuntu) handle_debian ;; - rhel|centos) handle_redhat ;; - sles|opensuse-leap) handle_suse ;; - *) - # Fallback for systems lacking /etc/os-release. - if [[ -f /etc/debian_version ]]; then - ID='debian' - handle_debian - elif [[ -f /etc/redhat-release ]]; then - ID='rhel' - handle_redhat - elif [[ -f /etc/SuSE-release ]]; then - ID='sles' - handle_suse - else - fail "Unidentifiable or unsupported platform. See -${AGENT_SUPPORTED_URL} for a list of supported platforms." - fi - esac - - if [[ "${ACTIONS[*]}" == *add-repo* ]]; then - resolve_version - add_repo - fi - if [[ "${ACTIONS[*]}" == *also-install* ]]; then - install_agent - elif [[ "${ACTIONS[*]}" == *uninstall* ]]; then - uninstall_agent - fi - if [[ "${ACTIONS[*]}" == *remove-repo* ]]; then - remove_repo - fi - - if [[ "${CHANGED}" == 0 ]]; then - echo 'No changes made.' - fi - - if [[ -n "${DRY_RUN:-}" ]]; then - echo 'Finished dry run. This was only a simulation, remove the --dry-run flag -to perform an actual execution of the script.' - fi -} - -main "$@" From b02f48ab19143a2aa098010e12b9cd3518d877f7 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 1 Apr 2022 15:09:22 +0000 Subject: [PATCH 07/11] chore: try different dir name --- .github/workflows/main.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 06d5e125..98acf841 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -28,5 +28,4 @@ jobs: steps: - name: Download script. id: update-add-logging-agent-repo - run: wget -O ./gce/add-logging-agent-repo.sh https://dl.google.com/cloudagents/add-logging-agent-repo.sh - \ No newline at end of file + run: wget -O gce/add-logging-agent-repo.sh https://dl.google.com/cloudagents/add-logging-agent-repo.sh From 0481206dbdeb885dcdfd8ec0e357b55725d4f621 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 1 Apr 2022 15:37:09 +0000 Subject: [PATCH 08/11] chore: clean up workflows file, fix startup script --- .github/workflows/main.yaml | 31 -- gce/add-logging-agent-repo.sh | 525 ++++++++++++++++++++++++++++++++++ gce/startup-script.sh | 16 +- 3 files changed, 533 insertions(+), 39 deletions(-) delete mode 100644 .github/workflows/main.yaml diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml deleted file mode 100644 index 98acf841..00000000 --- a/.github/workflows/main.yaml +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright 2022 Google LLC - -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at - -# https://www.apache.org/licenses/LICENSE-2.0 - -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -name: A workflow for updating the add-logging-agent-repo.sh script. -# Controls when the action will run. - -on: workflow_dispatch - # schedule: - # # * is a special character in YAML so you have to quote this string - # # Run this Github Action every Tuesday at 7 AM UTC - # - cron: '0 7 * * 2' - -jobs: - build: - name: Download latest add-logging-agent-repo.sh script. - runs-on: ubuntu-latest - steps: - - name: Download script. - id: update-add-logging-agent-repo - run: wget -O gce/add-logging-agent-repo.sh https://dl.google.com/cloudagents/add-logging-agent-repo.sh diff --git a/gce/add-logging-agent-repo.sh b/gce/add-logging-agent-repo.sh index e69de29b..ae08a405 100644 --- a/gce/add-logging-agent-repo.sh +++ b/gce/add-logging-agent-repo.sh @@ -0,0 +1,525 @@ +#!/bin/bash +# Copyright 2020 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# *NOTE*: The source of truth for this script is https://dl.google.com/cloudagents/add-logging-agent-repo.sh +# It has been committed to this repository to follow security best practices. +# +# Add repository for the Google logging agent. +# +# This script adds the required apt or yum repository and installs or uninstalls +# the agent based on the corresponding flags. +# +# Available flags: +# * `--verbose`: +# Turns on verbose logging during the script execution, which is helpful for +# debugging purposes. +# +# * `--also-install`: +# Installs the agent after adding the agent package repository. If this flag +# is absent, the script only adds the agent package repository. This flag +# can not be run with the `--uninstall` flag. +# +# * `--version `: +# Sets the agent version for the script to install. Allowed formats: +# * `latest`: +# Adds an agent package repository that contains all agent versions, and +# installs the latest version of the agent. +# * `MAJOR_VERSION.*.*`: +# Adds an agent package repository that contains all agent versions up to +# this major version (e.g. `1.*.*`), and installs the latest version of +# the agent within the range of that major version. +# * `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION`: +# Adds an agent package repository that contains all agent versions, and +# installs the specified version of the agent (e.g. `3.2.1`). +# +# * `--uninstall`: +# Uninstalls the agent. This flag can not be run with the `--also-install` +# flag. +# +# * `--remove-repo`: +# Removes the corresponding agent package repository after installing or +# uninstalling the agent. +# +# * `--dry-run`: +# Triggers only a dry run of the script execution and prints out the +# commands that it is supposed to execute. This is helpful to know what +# actions the script will take. +# +# * `--structured`: +# When installing or uninstalling packages, use the structured catch-all +# config package. +# +# * `--unstructured`: +# When installing or uninstalling packages, use the unstructured catch-all +# config package. This is currently the default. +# +# Sample usage: +# * To add the repo that contains all agent versions, run: +# $ bash add-logging-agent-repo.sh +# +# * To add the repo and also install the agent, run: +# $ bash add-logging-agent-repo.sh --also-install --version= +# +# * To uninstall the agent run: +# $ bash add-logging-agent-repo.sh --uninstall +# +# * To uninstall the agent and remove the repo, run: +# $ bash add-logging-agent-repo.sh --uninstall --remove-repo +# +# * To run the script with verbose logging, run: +# $ bash add-logging-agent-repo.sh --also-install --verbose +# +# * To run the script in dry-run mode, run: +# $ bash add-logging-agent-repo.sh --also-install --dry-run +# +# The environment variable `DO_NOT_INSTALL_CATCH_ALL_CONFIG` can be set to +# prevent the google-fluentd-catch-all-config package from being installed. +# +# Internal usage only: +# The environment variable `REPO_SUFFIX` can be set to alter which repository is +# used. A dash (-) will be inserted prior to the supplied suffix. `REPO_SUFFIX` +# defaults to `all` which contains all agent versions across different major +# versions. The full repository name is: +# "google-cloud-logging-[-]-". + +# Ignore the return code of command substitution in variables. +# shellcheck disable=SC2155 +# +# Initialize var used to notify config management tools of when a change is made. +CHANGED=0 + +fail() { + echo >&2 "[$(date +'%Y-%m-%dT%H:%M:%S%z')] $*" + exit 1 +} + +# Parsing flag value. +declare -a ACTIONS=() +DRY_RUN='' +VERBOSE='false' +declare -a STRUCTURED_FLAGS=() +while getopts -- '-:' OPTCHAR; do + case "${OPTCHAR}" in + -) + case "${OPTARG}" in + # Note: Do not remove entries from this list when deprecating flags. + # That would break user scripts that specify those flags. Instead, + # leave the flag in place but make it a noop. + also-install) ACTIONS+=('also-install') ;; + version=*) AGENT_VERSION="${OPTARG#*=}" ;; + uninstall) ACTIONS+=('uninstall') ;; + remove-repo) ACTIONS+=('remove-repo') ;; + dry-run) echo 'Starting dry run'; DRY_RUN='dryrun' ;; + verbose) VERBOSE='true' ;; + structured) STRUCTURED_FLAGS+=('true') ;; + unstructured) STRUCTURED_FLAGS+=('false') ;; + *) fail "Unknown option '${OPTARG}'." ;; + esac + esac +done +[[ "${ACTIONS[*]}" == *uninstall* || ( "${ACTIONS[*]}" == *remove-repo* && "${ACTIONS[*]}" != *also-install* )]] || \ + ACTIONS+=('add-repo') +# Sort the actions array for easier parsing. +readarray -t ACTIONS < <(printf '%s\n' "${ACTIONS[@]}" | sort) +readonly ACTIONS DRY_RUN VERBOSE +# Sort the structured flags array for easier parsing. +readarray -t STRUCTURED_FLAGS < <(printf '%s\n' "${STRUCTURED_FLAGS[@]}" | sort) +readonly STRUCTURED_FLAGS + +if [[ "${ACTIONS[*]}" == *also-install*uninstall* ]]; then + fail "Received conflicting flags 'also-install' and 'uninstall'." +fi + +if [[ "${STRUCTURED_FLAGS[*]}" == *false*true* ]]; then + fail "Received conflicting flags 'structured' and 'unstructured'." +fi + +if [[ ! ("${ACTIONS[*]}" == *also-install* || "${ACTIONS[*]}" == *uninstall* ) && -n "${STRUCTURED_FLAGS[*]}" ]]; then + fail "The 'structured' and 'unstructured' flags are only used in 'also-install' or 'uninstall' mode." +fi + +if [[ "${VERBOSE}" == 'true' ]]; then + echo 'Enable verbose logging.' + set -x +fi + +# Host that serves the repositories. +REPO_HOST='packages.cloud.google.com' + +# URL for the logging agent documentation. +AGENT_DOCS_URL='https://cloud.google.com/logging/docs/agent' + +# URL documentation which lists supported platforms for running the logging agent. +AGENT_SUPPORTED_URL="${AGENT_DOCS_URL}/#agent-os-list" + +# Packages to install. +AGENT_PACKAGE='google-fluentd' +if [[ -z "${DO_NOT_INSTALL_CATCH_ALL_CONFIG:-}" ]]; then + if [[ "${STRUCTURED_FLAGS[*]}" == *true* ]]; then + declare -a ADDITIONAL_PACKAGES=('google-fluentd-catch-all-config-structured') + else + declare -a ADDITIONAL_PACKAGES=('google-fluentd-catch-all-config') + fi +fi + +if [[ -f /etc/os-release ]]; then + . /etc/os-release +fi + +# If dry-run mode is enabled, echo VM state-changing commands instead of executing them. +dryrun() { + # Needed for commands that use pipes. + if [[ ! -t 0 ]]; then + cat + fi + printf -v cmd_str '%q ' "$@" + echo "DRY_RUN: Not executing '$cmd_str'" +} + +refresh_failed() { + local REPO_TYPE="$1" + local OS_FAMILY="$2" + fail "Could not refresh the google-cloud-logging ${REPO_TYPE} repositories. +Please check your network connectivity and make sure you are running a supported +${OS_FAMILY} distribution. See ${AGENT_SUPPORTED_URL} +for a list of supported platforms." +} + +resolve_version() { + if [[ "${AGENT_VERSION:-latest}" == 'latest' ]]; then + AGENT_VERSION='' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + REPO_SUFFIX="${REPO_SUFFIX:-"${AGENT_VERSION%%.*}"}" + elif ! grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + fail "The agent version [${AGENT_VERSION}] is not allowed. Expected values: [latest], +or anything in the format of [MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION] or [MAJOR_VERSION.*.*]." + fi +} + +handle_debian() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-oDebug::pkgAcquire::Worker=1) + + add_repo() { + [[ -n "${REPO_CODENAME:-}" ]] || lsb_release -v >/dev/null 2>&1 || { \ + apt-get update; apt-get -y install lsb-release; CHANGED=1; + } + [[ "$(dpkg -l apt-transport-https 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install apt-transport-https; CHANGED=1; + } + [[ "$(dpkg -l ca-certificates 2>&1 | grep -o '^[a-z][a-z]')" == 'ii' ]] || { \ + ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install ca-certificates; CHANGED=1; + } + local CODENAME="${REPO_CODENAME:-"$(lsb_release -sc)"}" + local REPO_NAME="google-cloud-logging-${CODENAME}-${REPO_SUFFIX:-all}" + local REPO_DATA="deb https://${REPO_HOST}/apt ${REPO_NAME} main" + if ! cmp -s <<<"${REPO_DATA}" - /etc/apt/sources.list.d/google-cloud-logging.list; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/apt/sources.list.d/google-cloud-logging.list + ${DRY_RUN} curl --connect-timeout 5 -s -f "https://${REPO_HOST}/apt/doc/apt-key.gpg" \ + | ${DRY_RUN} apt-key add - + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/apt/sources.list.d/google-cloud-logging.list ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/apt/sources.list.d/google-cloud-logging.list + CHANGED=1 + fi + } + + expected_version_installed() { + [[ "$(dpkg -l "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" 2>&1 | grep -o '^[a-z][a-z]' | sort -u)" == 'ii' ]] || \ + return + if [[ -z "${AGENT_VERSION:-}" ]]; then + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" && \ + apt-get --dry-run install "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qo '^0 upgraded, 0 newly installed' + else + dpkg -l "${AGENT_PACKAGE}" | grep -qE "$AGENT_PACKAGE $AGENT_VERSION" + fi + } + + install_agent() { + ${DRY_RUN} apt-get update || refresh_failed 'apt' "${ID}" + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # apt package version format: e.g. 1.8.0-1. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="=${AGENT_VERSION}-*" + else + AGENT_VERSION="=${AGENT_VERSION%.\*}" + fi + fi + ${DRY_RUN} apt-get -y --allow-downgrades "${EXTRA_OPTS[@]}" install "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall_agent() { + # Return early unless at least one package is installed. + dpkg -l "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" 2>&1 | grep -qo '^ii' || return + ${DRY_RUN} apt-get -y "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" || \ + fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_rpm() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-v) + + add_repo() { + local REPO_NAME="google-cloud-logging-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + local REPO_DATA="\ +[google-cloud-logging] +name=Google Cloud Logging Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgcheck=1 +repo_gpgcheck=0 +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/yum.repos.d/google-cloud-logging.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/yum.repos.d/google-cloud-logging.repo + # After repo upgrades, CentOS7/RHEL7 won't pick up newly available packages + # until the cache is cleared. + ${DRY_RUN} rm -rf /var/cache/yum/*/*/google-cloud-logging/ + CHANGED=1 + fi + } + + remove_repo() { + if [[ -f /etc/yum.repos.d/google-cloud-logging.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/yum.repos.d/google-cloud-logging.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + grep -qE "${AGENT_VERSION}" <<<"${CURRENT_VERSION}" && \ + yum -y check-update "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 + else + CURRENT_VERSION="$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" + [[ "${AGENT_VERSION}" == "${CURRENT_VERSION}" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + ${DRY_RUN} yum -y list updates || refresh_failed 'yum' "${ID}" + local COMMAND='install' + if [[ -n "${AGENT_VERSION:-}" ]]; then + [[ -z "${CURRENT_VERSION:-}" ]] || \ + [[ "${AGENT_VERSION}" == "$(sort -rV <<<"${AGENT_VERSION}"$'\n'"${CURRENT_VERSION}" | head -1)" ]] || \ + COMMAND='downgrade' + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # yum package version format: e.g. 1.0.1-1.el8. + if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="-${AGENT_VERSION}-1*" + else + AGENT_VERSION="-${AGENT_VERSION}" + fi + fi + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" "${COMMAND}" "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall_agent() { + # Return early if none of the packages are installed. + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" || \ + fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." + CHANGED=1 + } +} + +handle_redhat() { + local MAJOR_VERSION="$(rpm --eval %{?rhel})" + CODENAME="el${MAJOR_VERSION}" + handle_rpm +} + +handle_amazon_linux() { + CODENAME='el6' + handle_rpm +} + +handle_suse() { + declare -a EXTRA_OPTS=() + [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-vv) + + add_repo() { + local SUSE_VERSION=${VERSION_ID%%.*} + local CODENAME="sles${SUSE_VERSION}" + local REPO_NAME="google-cloud-logging-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + { + ${DRY_RUN} zypper --non-interactive refresh || { \ + echo >&2 'Could not refresh zypper repositories.'; \ + echo >&2 'This is not necessarily a fatal error; proceeding...'; \ + } + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + local REPO_DATA="\ +[google-cloud-logging] +name=Google Cloud Logging Agent Repository +baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} +autorefresh=0 +enabled=1 +type=rpm-md +gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg + https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if ! cmp -s <<<"${REPO_DATA}" - /etc/zypp/repos.d/google-cloud-logging.repo; then + echo "Adding agent repository for ${ID}." + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/zypp/repos.d/google-cloud-logging.repo + CHANGED=1 + fi + local RPM_KEYS="$(rpm --query gpg-pubkey)" # Save the installed keys. + ${DRY_RUN} rpm --import "https://${REPO_HOST}/yum/doc/yum-key.gpg" "https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" + if [[ -n "${DRY_RUN:-}" ]] || ! cmp --silent <<<"${RPM_KEYS}" - <(rpm --query gpg-pubkey); then + CHANGED=1 + fi + { + ${DRY_RUN} zypper --non-interactive --gpg-auto-import-keys refresh google-cloud-logging || \ + refresh_failed 'zypper' "${ID}"; \ + } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 + } + + remove_repo() { + if [[ -f /etc/zypp/repos.d/google-cloud-logging.repo ]]; then + echo "Removing agent repository for ${ID}." + ${DRY_RUN} rm /etc/zypp/repos.d/google-cloud-logging.repo + CHANGED=1 + fi + } + + expected_version_installed() { + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" >/dev/null 2>&1 || return + if [[ -z "${AGENT_VERSION:-}" ]]; then + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + elif grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}" | grep -qE "${AGENT_VERSION}" && \ + zypper --non-interactive update --dry-run "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" \ + | grep -qE '^Nothing to do.' + else + [[ "${AGENT_VERSION}" == "$(rpm -q --queryformat '%{VERSION}' "${AGENT_PACKAGE}")" ]] + fi + } + + install_agent() { + expected_version_installed || { \ + if [[ -n "${AGENT_VERSION:-}" ]]; then + # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. + # zypper package version format: e.g. 1.0.6-1.sles15. + if grep -qE '^[0-9]+\.\*\.\*$' <<<"${AGENT_VERSION}"; then + AGENT_VERSION="<$(( ${AGENT_VERSION%%.*} + 1 ))" + else + AGENT_VERSION="=${AGENT_VERSION}" + fi + fi + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" install --oldpackage "${AGENT_PACKAGE}${AGENT_VERSION}" \ + "${ADDITIONAL_PACKAGES[@]}" || fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} \ +installation failed." + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} installation succeeded." + CHANGED=1 + } + } + + uninstall_agent() { + # Return early if none of the packages are installed. + rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" || \ + fail "${AGENT_PACKAGE} uninstallation failed." + # zypper doesn't like removing packages that are not installed. + if rpm -q "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$'; then + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${ADDITIONAL_PACKAGES[@]}" || \ + fail "${ADDITIONAL_PACKAGES[*]} uninstallation failed." + fi + echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." + CHANGED=1 + } +} + +main() { + case "${ID:-}" in + amzn) handle_amazon_linux ;; + debian|ubuntu) handle_debian ;; + rhel|centos) handle_redhat ;; + sles|opensuse-leap) handle_suse ;; + *) + # Fallback for systems lacking /etc/os-release. + if [[ -f /etc/debian_version ]]; then + ID='debian' + handle_debian + elif [[ -f /etc/redhat-release ]]; then + ID='rhel' + handle_redhat + elif [[ -f /etc/SuSE-release ]]; then + ID='sles' + handle_suse + else + fail "Unidentifiable or unsupported platform. See +${AGENT_SUPPORTED_URL} for a list of supported platforms." + fi + esac + + if [[ "${ACTIONS[*]}" == *add-repo* ]]; then + resolve_version + add_repo + fi + if [[ "${ACTIONS[*]}" == *also-install* ]]; then + install_agent + elif [[ "${ACTIONS[*]}" == *uninstall* ]]; then + uninstall_agent + fi + if [[ "${ACTIONS[*]}" == *remove-repo* ]]; then + remove_repo + fi + + if [[ "${CHANGED}" == 0 ]]; then + echo 'No changes made.' + fi + + if [[ -n "${DRY_RUN:-}" ]]; then + echo 'Finished dry run. This was only a simulation, remove the --dry-run flag +to perform an actual execution of the script.' + fi +} + +main "$@" diff --git a/gce/startup-script.sh b/gce/startup-script.sh index d18e0c54..8937a111 100644 --- a/gce/startup-script.sh +++ b/gce/startup-script.sh @@ -12,22 +12,22 @@ # See the License for the specific language governing permissions and # limitations under the License. -# Echo commands -set -v +# Echo commands and fail on error +set -ev # [START getting_started_gce_startup_script] +# Install or update needed software +apt-get update +apt-get install -yq git supervisor python python-pip python3-distutils +pip install --upgrade pip virtualenv + # Fetch source code export HOME=/root -git clone https://github.com/GoogleCloudPlatform/getting-started-python.git /opt/app +git clone https://github.com/busunkim96/getting-started-python.git /opt/app # Install Cloud Logging agent sudo bash /opt/app/gce/add-logging-agent-repo.sh --also-install -# Install or update needed software -apt-get update -apt-get install -yq git supervisor python python-pip python3-distutils -pip install --upgrade pip virtualenv - # Account to own server process useradd -m -d /home/pythonapp pythonapp From 36c5834f8b53563c6070fb472ea7c23fd288002d Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 1 Apr 2022 15:45:41 +0000 Subject: [PATCH 09/11] chore: use ops agent --- ....sh => add-google-cloud-ops-agent-repo.sh} | 205 ++++++++---------- gce/startup-script.sh | 4 +- 2 files changed, 98 insertions(+), 111 deletions(-) rename gce/{add-logging-agent-repo.sh => add-google-cloud-ops-agent-repo.sh} (72%) diff --git a/gce/add-logging-agent-repo.sh b/gce/add-google-cloud-ops-agent-repo.sh similarity index 72% rename from gce/add-logging-agent-repo.sh rename to gce/add-google-cloud-ops-agent-repo.sh index ae08a405..3b1f3be8 100644 --- a/gce/add-logging-agent-repo.sh +++ b/gce/add-google-cloud-ops-agent-repo.sh @@ -13,10 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. # -# *NOTE*: The source of truth for this script is https://dl.google.com/cloudagents/add-logging-agent-repo.sh -# It has been committed to this repository to follow security best practices. -# -# Add repository for the Google logging agent. +# Add repository for the Google ops agent. # # This script adds the required apt or yum repository and installs or uninstalls # the agent based on the corresponding flags. @@ -57,42 +54,40 @@ # commands that it is supposed to execute. This is helpful to know what # actions the script will take. # -# * `--structured`: -# When installing or uninstalling packages, use the structured catch-all -# config package. +# * `--uninstall-standalone-logging-agent`: +# Uninstalls the standalone logging agent (`google-fluentd`). # -# * `--unstructured`: -# When installing or uninstalling packages, use the unstructured catch-all -# config package. This is currently the default. +# * `--uninstall-standalone-monitoring-agent`: +# Uninstalls the standalone monitoring agent (`stackdriver-agent`). # # Sample usage: # * To add the repo that contains all agent versions, run: -# $ bash add-logging-agent-repo.sh +# $ bash add-google-cloud-ops-agent-repo.sh # # * To add the repo and also install the agent, run: -# $ bash add-logging-agent-repo.sh --also-install --version= +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --version= # # * To uninstall the agent run: -# $ bash add-logging-agent-repo.sh --uninstall +# $ bash add-google-cloud-ops-agent-repo.sh --uninstall # # * To uninstall the agent and remove the repo, run: -# $ bash add-logging-agent-repo.sh --uninstall --remove-repo +# $ bash add-google-cloud-ops-agent-repo.sh --uninstall --remove-repo # # * To run the script with verbose logging, run: -# $ bash add-logging-agent-repo.sh --also-install --verbose +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --verbose # # * To run the script in dry-run mode, run: -# $ bash add-logging-agent-repo.sh --also-install --dry-run +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --dry-run # -# The environment variable `DO_NOT_INSTALL_CATCH_ALL_CONFIG` can be set to -# prevent the google-fluentd-catch-all-config package from being installed. +# * To replace standalone agents with the Ops agent, run: +# $ bash add-google-cloud-ops-agent-repo.sh --also-install --uninstall-standalone-logging-agent --uninstall-standalone-monitoring-agent # # Internal usage only: # The environment variable `REPO_SUFFIX` can be set to alter which repository is # used. A dash (-) will be inserted prior to the supplied suffix. `REPO_SUFFIX` # defaults to `all` which contains all agent versions across different major # versions. The full repository name is: -# "google-cloud-logging-[-]-". +# "google-cloud-ops-agent-[-]-". # Ignore the return code of command substitution in variables. # shellcheck disable=SC2155 @@ -109,7 +104,6 @@ fail() { declare -a ACTIONS=() DRY_RUN='' VERBOSE='false' -declare -a STRUCTURED_FLAGS=() while getopts -- '-:' OPTCHAR; do case "${OPTCHAR}" in -) @@ -121,33 +115,22 @@ while getopts -- '-:' OPTCHAR; do version=*) AGENT_VERSION="${OPTARG#*=}" ;; uninstall) ACTIONS+=('uninstall') ;; remove-repo) ACTIONS+=('remove-repo') ;; + uninstall-standalone-logging-agent) ACTIONS+=('uninstall-standalone-logging-agent') ;; + uninstall-standalone-monitoring-agent) ACTIONS+=('uninstall-standalone-monitoring-agent') ;; dry-run) echo 'Starting dry run'; DRY_RUN='dryrun' ;; verbose) VERBOSE='true' ;; - structured) STRUCTURED_FLAGS+=('true') ;; - unstructured) STRUCTURED_FLAGS+=('false') ;; *) fail "Unknown option '${OPTARG}'." ;; esac esac done -[[ "${ACTIONS[*]}" == *uninstall* || ( "${ACTIONS[*]}" == *remove-repo* && "${ACTIONS[*]}" != *also-install* )]] || \ +[[ " ${ACTIONS[*]} " == *\ uninstall\ * || ( " ${ACTIONS[*]} " == *\ remove-repo\ * && " ${ACTIONS[*]} " != *\ also-install\ * )]] || \ ACTIONS+=('add-repo') # Sort the actions array for easier parsing. readarray -t ACTIONS < <(printf '%s\n' "${ACTIONS[@]}" | sort) readonly ACTIONS DRY_RUN VERBOSE -# Sort the structured flags array for easier parsing. -readarray -t STRUCTURED_FLAGS < <(printf '%s\n' "${STRUCTURED_FLAGS[@]}" | sort) -readonly STRUCTURED_FLAGS - -if [[ "${ACTIONS[*]}" == *also-install*uninstall* ]]; then - fail "Received conflicting flags 'also-install' and 'uninstall'." -fi - -if [[ "${STRUCTURED_FLAGS[*]}" == *false*true* ]]; then - fail "Received conflicting flags 'structured' and 'unstructured'." -fi -if [[ ! ("${ACTIONS[*]}" == *also-install* || "${ACTIONS[*]}" == *uninstall* ) && -n "${STRUCTURED_FLAGS[*]}" ]]; then - fail "The 'structured' and 'unstructured' flags are only used in 'also-install' or 'uninstall' mode." +if [[ " ${ACTIONS[*]} " == *\ also-install*uninstall\ * ]]; then + fail "Received conflicting flags 'also-install' and 'uninstall'." fi if [[ "${VERBOSE}" == 'true' ]]; then @@ -158,21 +141,15 @@ fi # Host that serves the repositories. REPO_HOST='packages.cloud.google.com' -# URL for the logging agent documentation. -AGENT_DOCS_URL='https://cloud.google.com/logging/docs/agent' +# URL for the ops agent documentation. +AGENT_DOCS_URL='https://cloud.google.com/stackdriver/docs/solutions/ops-agent' -# URL documentation which lists supported platforms for running the logging agent. -AGENT_SUPPORTED_URL="${AGENT_DOCS_URL}/#agent-os-list" +# URL documentation which lists supported platforms for running the ops agent. +AGENT_SUPPORTED_URL="${AGENT_DOCS_URL}/#supported_operating_systems" # Packages to install. -AGENT_PACKAGE='google-fluentd' -if [[ -z "${DO_NOT_INSTALL_CATCH_ALL_CONFIG:-}" ]]; then - if [[ "${STRUCTURED_FLAGS[*]}" == *true* ]]; then - declare -a ADDITIONAL_PACKAGES=('google-fluentd-catch-all-config-structured') - else - declare -a ADDITIONAL_PACKAGES=('google-fluentd-catch-all-config') - fi -fi +AGENT_PACKAGE='google-cloud-ops-agent' +declare -a ADDITIONAL_PACKAGES=() if [[ -f /etc/os-release ]]; then . /etc/os-release @@ -191,7 +168,7 @@ dryrun() { refresh_failed() { local REPO_TYPE="$1" local OS_FAMILY="$2" - fail "Could not refresh the google-cloud-logging ${REPO_TYPE} repositories. + fail "Could not refresh the google-cloud-ops-agent ${REPO_TYPE} repositories. Please check your network connectivity and make sure you are running a supported ${OS_FAMILY} distribution. See ${AGENT_SUPPORTED_URL} for a list of supported platforms." @@ -223,11 +200,11 @@ handle_debian() { ${DRY_RUN} apt-get update; ${DRY_RUN} apt-get -y install ca-certificates; CHANGED=1; } local CODENAME="${REPO_CODENAME:-"$(lsb_release -sc)"}" - local REPO_NAME="google-cloud-logging-${CODENAME}-${REPO_SUFFIX:-all}" + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-${REPO_SUFFIX:-all}" local REPO_DATA="deb https://${REPO_HOST}/apt ${REPO_NAME} main" - if ! cmp -s <<<"${REPO_DATA}" - /etc/apt/sources.list.d/google-cloud-logging.list; then + if ! cmp -s <<<"${REPO_DATA}" - /etc/apt/sources.list.d/google-cloud-ops-agent.list; then echo "Adding agent repository for ${ID}." - ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/apt/sources.list.d/google-cloud-logging.list + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/apt/sources.list.d/google-cloud-ops-agent.list ${DRY_RUN} curl --connect-timeout 5 -s -f "https://${REPO_HOST}/apt/doc/apt-key.gpg" \ | ${DRY_RUN} apt-key add - CHANGED=1 @@ -235,9 +212,9 @@ handle_debian() { } remove_repo() { - if [[ -f /etc/apt/sources.list.d/google-cloud-logging.list ]]; then + if [[ -f /etc/apt/sources.list.d/google-cloud-ops-agent.list ]]; then echo "Removing agent repository for ${ID}." - ${DRY_RUN} rm /etc/apt/sources.list.d/google-cloud-logging.list + ${DRY_RUN} rm /etc/apt/sources.list.d/google-cloud-ops-agent.list CHANGED=1 fi } @@ -262,9 +239,9 @@ handle_debian() { expected_version_installed || { \ if [[ -n "${AGENT_VERSION:-}" ]]; then # Differentiate `MAJOR_VERSION.MINOR_VERSION.PATCH_VERSION` from `MAJOR_VERSION.*.*`. - # apt package version format: e.g. 1.8.0-1. + # apt package version format: e.g. 2.0.1~debian9.13. if grep -qE '^[0-9]+\.[0-9]+\.[0-9]+$' <<<"${AGENT_VERSION}"; then - AGENT_VERSION="=${AGENT_VERSION}-*" + AGENT_VERSION="=${AGENT_VERSION}~*" else AGENT_VERSION="=${AGENT_VERSION%.\*}" fi @@ -277,13 +254,14 @@ installation failed." } } - uninstall_agent() { - # Return early unless at least one package is installed. - dpkg -l "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" 2>&1 | grep -qo '^ii' || return - ${DRY_RUN} apt-get -y "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" || \ - fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation failed." - echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." - CHANGED=1 + uninstall() { + local -a packages=("$@") + # Return early unless at least one package is installed. + dpkg -l "${packages[@]}" 2>&1 | grep -qo '^ii' || return + ${DRY_RUN} apt-get -y "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 } } @@ -292,10 +270,10 @@ handle_rpm() { [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-v) add_repo() { - local REPO_NAME="google-cloud-logging-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" local REPO_DATA="\ -[google-cloud-logging] -name=Google Cloud Logging Agent Repository +[google-cloud-ops-agent] +name=Google Cloud Ops Agent Repository baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} autorefresh=0 enabled=1 @@ -304,20 +282,20 @@ gpgcheck=1 repo_gpgcheck=0 gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" - if ! cmp -s <<<"${REPO_DATA}" - /etc/yum.repos.d/google-cloud-logging.repo; then + if ! cmp -s <<<"${REPO_DATA}" - /etc/yum.repos.d/google-cloud-ops-agent.repo; then echo "Adding agent repository for ${ID}." - ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/yum.repos.d/google-cloud-logging.repo + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/yum.repos.d/google-cloud-ops-agent.repo # After repo upgrades, CentOS7/RHEL7 won't pick up newly available packages # until the cache is cleared. - ${DRY_RUN} rm -rf /var/cache/yum/*/*/google-cloud-logging/ + ${DRY_RUN} rm -rf /var/cache/yum/*/*/google-cloud-ops-agent/ CHANGED=1 fi } remove_repo() { - if [[ -f /etc/yum.repos.d/google-cloud-logging.repo ]]; then + if [[ -f /etc/yum.repos.d/google-cloud-ops-agent.repo ]]; then echo "Removing agent repository for ${ID}." - ${DRY_RUN} rm /etc/yum.repos.d/google-cloud-logging.repo + ${DRY_RUN} rm /etc/yum.repos.d/google-cloud-ops-agent.repo CHANGED=1 fi } @@ -360,13 +338,14 @@ installation failed." } } - uninstall_agent() { - # Return early if none of the packages are installed. - rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$' || return - ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" || \ - fail "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation failed." - echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." - CHANGED=1 + uninstall() { + local -a packages=("$@") + # Return early if none of the packages are installed. + rpm -q "${packages[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} yum -y "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 } } @@ -376,11 +355,6 @@ handle_redhat() { handle_rpm } -handle_amazon_linux() { - CODENAME='el6' - handle_rpm -} - handle_suse() { declare -a EXTRA_OPTS=() [[ "${VERBOSE}" == 'true' ]] && EXTRA_OPTS+=(-vv) @@ -388,7 +362,7 @@ handle_suse() { add_repo() { local SUSE_VERSION=${VERSION_ID%%.*} local CODENAME="sles${SUSE_VERSION}" - local REPO_NAME="google-cloud-logging-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" + local REPO_NAME="google-cloud-ops-agent-${CODENAME}-\$basearch-${REPO_SUFFIX:-all}" { ${DRY_RUN} zypper --non-interactive refresh || { \ echo >&2 'Could not refresh zypper repositories.'; \ @@ -396,17 +370,17 @@ handle_suse() { } } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 local REPO_DATA="\ -[google-cloud-logging] -name=Google Cloud Logging Agent Repository +[google-cloud-ops-agent] +name=Google Cloud Ops Agent Repository baseurl=https://${REPO_HOST}/yum/repos/${REPO_NAME} autorefresh=0 enabled=1 type=rpm-md gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg https://${REPO_HOST}/yum/doc/rpm-package-key.gpg" - if ! cmp -s <<<"${REPO_DATA}" - /etc/zypp/repos.d/google-cloud-logging.repo; then + if ! cmp -s <<<"${REPO_DATA}" - /etc/zypp/repos.d/google-cloud-ops-agent.repo; then echo "Adding agent repository for ${ID}." - ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/zypp/repos.d/google-cloud-logging.repo + ${DRY_RUN} tee <<<"${REPO_DATA}" /etc/zypp/repos.d/google-cloud-ops-agent.repo CHANGED=1 fi local RPM_KEYS="$(rpm --query gpg-pubkey)" # Save the installed keys. @@ -415,15 +389,15 @@ gpgkey=https://${REPO_HOST}/yum/doc/yum-key.gpg CHANGED=1 fi { - ${DRY_RUN} zypper --non-interactive --gpg-auto-import-keys refresh google-cloud-logging || \ + ${DRY_RUN} zypper --non-interactive --gpg-auto-import-keys refresh google-cloud-ops-agent || \ refresh_failed 'zypper' "${ID}"; \ } | grep -qF 'Retrieving repository' || [[ -n "${DRY_RUN:-}" ]] && CHANGED=1 } remove_repo() { - if [[ -f /etc/zypp/repos.d/google-cloud-logging.repo ]]; then + if [[ -f /etc/zypp/repos.d/google-cloud-ops-agent.repo ]]; then echo "Removing agent repository for ${ID}." - ${DRY_RUN} rm /etc/zypp/repos.d/google-cloud-logging.repo + ${DRY_RUN} rm /etc/zypp/repos.d/google-cloud-ops-agent.repo CHANGED=1 fi } @@ -461,24 +435,26 @@ installation failed." } } - uninstall_agent() { - # Return early if none of the packages are installed. - rpm -q "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$' || return - ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${AGENT_PACKAGE}" || \ - fail "${AGENT_PACKAGE} uninstallation failed." - # zypper doesn't like removing packages that are not installed. - if rpm -q "${ADDITIONAL_PACKAGES[@]}" | grep -qvE 'is not installed$'; then - ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${ADDITIONAL_PACKAGES[@]}" || \ - fail "${ADDITIONAL_PACKAGES[*]} uninstallation failed." - fi - echo "${AGENT_PACKAGE} ${ADDITIONAL_PACKAGES[*]} uninstallation succeeded." - CHANGED=1 + uninstall() { + local -a packages=("$@") + # Return early if none of the packages are installed. + rpm -q "${packages[@]}" | grep -qvE 'is not installed$' || return + ${DRY_RUN} zypper --non-interactive "${EXTRA_OPTS[@]}" remove "${packages[@]}" || \ + fail "${packages[*]} uninstallation failed." + echo "${packages[*]} uninstallation succeeded." + CHANGED=1 } } +save_configuration_files() { + local save_dir="/var/lib/google-cloud-ops-agent/saved_configs" + ${DRY_RUN} mkdir -p "${save_dir}" + ${DRY_RUN} cp -rp "$@" "${save_dir}" + echo "$* is now copied over to ${save_dir} folder." +} + main() { case "${ID:-}" in - amzn) handle_amazon_linux ;; debian|ubuntu) handle_debian ;; rhel|centos) handle_redhat ;; sles|opensuse-leap) handle_suse ;; @@ -499,16 +475,27 @@ ${AGENT_SUPPORTED_URL} for a list of supported platforms." fi esac - if [[ "${ACTIONS[*]}" == *add-repo* ]]; then + + if [[ " ${ACTIONS[*]} " == *\ uninstall-standalone-logging-agent\ * ]]; then + save_configuration_files "/etc/google-fluentd" + # This will also remove dependent packages, e.g. "google-fluentd-catch-all-config" or "google-fluentd-catch-all-config-structured". + uninstall "google-fluentd" + fi + if [[ " ${ACTIONS[*]} " == *\ uninstall-standalone-monitoring-agent\ * ]]; then + save_configuration_files "/etc/stackdriver" "/opt/stackdriver/collectd/etc" + uninstall "stackdriver-agent" + fi + if [[ " ${ACTIONS[*]} " == *\ add-repo\ * ]]; then resolve_version add_repo fi - if [[ "${ACTIONS[*]}" == *also-install* ]]; then + if [[ " ${ACTIONS[*]} " == *\ also-install\ * ]]; then install_agent - elif [[ "${ACTIONS[*]}" == *uninstall* ]]; then - uninstall_agent + elif [[ " ${ACTIONS[*]} " == *\ uninstall\ * ]]; then + save_configuration_files "/etc/google-cloud-ops-agent" + uninstall "${AGENT_PACKAGE}" "${ADDITIONAL_PACKAGES[@]}" fi - if [[ "${ACTIONS[*]}" == *remove-repo* ]]; then + if [[ " ${ACTIONS[*]} " == *\ remove-repo\ * ]]; then remove_repo fi diff --git a/gce/startup-script.sh b/gce/startup-script.sh index 8937a111..713da4e5 100644 --- a/gce/startup-script.sh +++ b/gce/startup-script.sh @@ -25,8 +25,8 @@ pip install --upgrade pip virtualenv export HOME=/root git clone https://github.com/busunkim96/getting-started-python.git /opt/app -# Install Cloud Logging agent -sudo bash /opt/app/gce/add-logging-agent-repo.sh --also-install +# Install Cloud Ops Agent +sudo bash /opt/app/gce/add-google-cloud-ops-agent-repo.sh --also-install # Account to own server process useradd -m -d /home/pythonapp pythonapp From 664bbbfebb7dd3389d3f28fc4cf6fd1fd77d5402 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 1 Apr 2022 15:48:55 +0000 Subject: [PATCH 10/11] chore: add note about copied file --- gce/add-google-cloud-ops-agent-repo.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gce/add-google-cloud-ops-agent-repo.sh b/gce/add-google-cloud-ops-agent-repo.sh index 3b1f3be8..63cf2dbd 100644 --- a/gce/add-google-cloud-ops-agent-repo.sh +++ b/gce/add-google-cloud-ops-agent-repo.sh @@ -13,6 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. # +# *NOTE*: The source of truth for this script is: +# https://dl.google.com/cloudagents/add-google-cloud-ops-agent-repo.sh +# See https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/installation +# for installation instructions. +# It is committed to this repository to follow security best practices. +# +# # Add repository for the Google ops agent. # # This script adds the required apt or yum repository and installs or uninstalls From 39bef9e56e16563c5f9fe939f8dbbafee63533d8 Mon Sep 17 00:00:00 2001 From: Bu Sun Kim Date: Fri, 1 Apr 2022 15:49:43 +0000 Subject: [PATCH 11/11] chore: fix source repo --- gce/startup-script.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gce/startup-script.sh b/gce/startup-script.sh index 713da4e5..cc35d30f 100644 --- a/gce/startup-script.sh +++ b/gce/startup-script.sh @@ -23,7 +23,7 @@ pip install --upgrade pip virtualenv # Fetch source code export HOME=/root -git clone https://github.com/busunkim96/getting-started-python.git /opt/app +git clone https://github.com/GoogleCloudPlatform/getting-started-python.git /opt/app # Install Cloud Ops Agent sudo bash /opt/app/gce/add-google-cloud-ops-agent-repo.sh --also-install