From 6a5c26a538af66b3ca8c82c3751e830da53613bd Mon Sep 17 00:00:00 2001 From: AbdelrahmanElawady Date: Fri, 28 Jul 2023 22:05:45 +0300 Subject: [PATCH] Add systemctl.sh to handle Varnish service and integrate Varnish with t3c-apply --- cache-config/t3c-apply/config/config.go | 3 + cache-config/t3c-apply/t3c-apply.go | 8 +- cache-config/t3c-apply/torequest/cmd.go | 1 + cache-config/t3c-apply/torequest/torequest.go | 30 ++++-- cache-config/t3c-generate/cfgfile/varnish.go | 15 ++- cache-config/t3c-generate/t3c-generate.go | 9 +- .../cdn-in-a-box/varnish/Dockerfile | 40 +++----- infrastructure/cdn-in-a-box/varnish/run.sh | 6 +- .../cdn-in-a-box/varnish/systemctl.sh | 94 +++++++++++++++++++ .../varnish/traffic_ops_ort.crontab | 18 ++++ lib/varnishcfg/vcl.go | 2 +- 11 files changed, 176 insertions(+), 50 deletions(-) create mode 100755 infrastructure/cdn-in-a-box/varnish/systemctl.sh create mode 100644 infrastructure/cdn-in-a-box/varnish/traffic_ops_ort.crontab diff --git a/cache-config/t3c-apply/config/config.go b/cache-config/t3c-apply/config/config.go index e7956e1eba..25b4df5c20 100644 --- a/cache-config/t3c-apply/config/config.go +++ b/cache-config/t3c-apply/config/config.go @@ -123,6 +123,7 @@ type Cfg struct { Version string GitRevision string LocalATSVersion string + CacheType string } func (cfg Cfg) AppVersion() string { return t3cutil.VersionStr(AppName, cfg.Version, cfg.GitRevision) } @@ -277,6 +278,7 @@ func GetCfg(appVersion string, gitRevision string) (Cfg, error) { defaultClientTLSVersions := getopt.StringLong("default-client-tls-versions", 'V', "", "Comma-delimited list of default TLS versions for Delivery Services with no Parameter, e.g. --default-tls-versions='1.1,1.2,1.3'. If omitted, all versions are enabled.") maxmindLocationPtr := getopt.StringLong("maxmind-location", 'M', "", "URL of a maxmind gzipped database file, to be installed into the trafficserver etc directory.") verbosePtr := getopt.CounterLong("verbose", 'v', `Log verbosity. Logging is output to stderr. By default, errors are logged. To log warnings, pass '-v'. To log info, pass '-vv'. To omit error logging, see '-s'`) + cache := getopt.StringLong("cache", 'T', "ats", "Cache server type. Generate configuration files for specific cache server type, e.g. 'ats', 'varnish'.") const silentFlagName = "silent" silentPtr := getopt.BoolLong(silentFlagName, 's', `Silent. Errors are not logged, and the 'verbose' flag is ignored. If a fatal error occurs, the return code will be non-zero but no text will be output to stderr`) @@ -612,6 +614,7 @@ If any of the related flags are also set, they override the mode's default behav Version: appVersion, GitRevision: gitRevision, LocalATSVersion: atsVersionStr, + CacheType: *cache, } if err = log.InitCfg(cfg); err != nil { diff --git a/cache-config/t3c-apply/t3c-apply.go b/cache-config/t3c-apply/t3c-apply.go index 24e1e8e91a..541ec334ba 100644 --- a/cache-config/t3c-apply/t3c-apply.go +++ b/cache-config/t3c-apply/t3c-apply.go @@ -220,7 +220,7 @@ func Main() int { } } else { - syncdsUpdate, err = trops.CheckSyncDSState(metaData) + syncdsUpdate, err = trops.CheckSyncDSState(metaData, cfg) if err != nil { log.Errorln("Checking syncds state: " + err.Error()) return GitCommitAndExit(ExitCodeSyncDSError, FailureExitMsg, cfg, metaData, oldMetaData) @@ -241,7 +241,7 @@ func Main() int { } else if rc == 0 { log.Infoln("updated the remap.config for reloading.") } - if err := trops.StartServices(&syncdsUpdate, metaData); err != nil { + if err := trops.StartServices(&syncdsUpdate, metaData, cfg); err != nil { log.Errorln("failed to start services: " + err.Error()) metaData.PartialSuccess = true return GitCommitAndExit(ExitCodeServicesError, PostConfigFailureExitMsg, cfg, metaData, oldMetaData) @@ -311,7 +311,7 @@ func Main() int { } } - if err := trops.StartServices(&syncdsUpdate, metaData); err != nil { + if err := trops.StartServices(&syncdsUpdate, metaData, cfg); err != nil { log.Errorln("failed to start services: " + err.Error()) metaData.PartialSuccess = true return GitCommitAndExit(ExitCodeServicesError, PostConfigFailureExitMsg, cfg, metaData, oldMetaData) @@ -373,7 +373,7 @@ func GitCommitAndExit(exitCode int, exitMsg string, cfg config.Cfg, metaData *t3 // so add the old files to the new metadata. // This is especially important for reval runs, which don't add all files. metaData.OwnedFilePaths = t3cutil.CombineOwnedFilePaths(metaData, oldMetaData) - if len(metaData.InstalledPackages) == 0 { + if len(metaData.InstalledPackages) == 0 && oldMetaData != nil { metaData.InstalledPackages = oldMetaData.InstalledPackages } WriteMetaData(cfg, metaData) diff --git a/cache-config/t3c-apply/torequest/cmd.go b/cache-config/t3c-apply/torequest/cmd.go index efc52a327c..597d19f358 100644 --- a/cache-config/t3c-apply/torequest/cmd.go +++ b/cache-config/t3c-apply/torequest/cmd.go @@ -72,6 +72,7 @@ func generate(cfg config.Cfg) ([]t3cutil.ATSConfigFile, error) { args := []string{ `generate`, "--dir=" + cfg.TsConfigDir, + "--cache=" + cfg.CacheType, } if cfg.LogLocationErr == log.LogLocationNull { diff --git a/cache-config/t3c-apply/torequest/torequest.go b/cache-config/t3c-apply/torequest/torequest.go index 5298091f6a..034d1e096b 100644 --- a/cache-config/t3c-apply/torequest/torequest.go +++ b/cache-config/t3c-apply/torequest/torequest.go @@ -742,7 +742,7 @@ func (r *TrafficOpsReq) CheckRevalidateState(sleepOverride bool) (UpdateStatus, // CheckSyncDSState retrieves and returns the DS Update status from Traffic Ops. // The metaData is this run's metadata. It must not be nil, and this function may add to it. -func (r *TrafficOpsReq) CheckSyncDSState(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) { +func (r *TrafficOpsReq) CheckSyncDSState(metaData *t3cutil.ApplyMetaData, cfg config.Cfg) (UpdateStatus, error) { updateStatus := UpdateTropsNotNeeded randDispSec := time.Duration(0) log.Debugln("Checking syncds state.") @@ -779,7 +779,7 @@ func (r *TrafficOpsReq) CheckSyncDSState(metaData *t3cutil.ApplyMetaData) (Updat } } else if !r.Cfg.IgnoreUpdateFlag { log.Errorln("no queued update needs to be applied. Running revalidation before exiting.") - r.RevalidateWhileSleeping(metaData) + r.RevalidateWhileSleeping(metaData, cfg) return UpdateTropsNotNeeded, nil } else { log.Errorln("Traffic Ops is signaling that no update is waiting to be applied.") @@ -1075,7 +1075,7 @@ func (r *TrafficOpsReq) ProcessPackagesWithMetaData(packageMetaData []string) er return nil } -func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) { +func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData, cfg config.Cfg) (UpdateStatus, error) { updateStatus, err := r.CheckRevalidateState(true) if err != nil { return updateStatus, err @@ -1099,7 +1099,7 @@ func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData) t3cutil.WriteActionLog(t3cutil.ActionLogActionUpdateFilesReval, t3cutil.ActionLogStatusSuccess, metaData) } - if err := r.StartServices(&updateStatus, metaData); err != nil { + if err := r.StartServices(&updateStatus, metaData, cfg); err != nil { return updateStatus, errors.New("failed to start services: " + err.Error()) } @@ -1116,7 +1116,7 @@ func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData) // StartServices reloads, restarts, or starts ATS as necessary, // according to the changed config files and run mode. // Returns nil on success or any error. -func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cutil.ApplyMetaData) error { +func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cutil.ApplyMetaData, cfg config.Cfg) error { serviceNeeds := t3cutil.ServiceNeedsNothing if r.Cfg.ServiceAction == t3cutil.ApplyServiceActionFlagRestart { serviceNeeds = t3cutil.ServiceNeedsRestart @@ -1138,13 +1138,17 @@ func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cu serviceNeeds = t3cutil.ServiceNeedsReload } } + packageName := "trafficserver" + if cfg.CacheType == "varnish" { + packageName = "varnish" + } - if (serviceNeeds == t3cutil.ServiceNeedsRestart || serviceNeeds == t3cutil.ServiceNeedsReload) && !r.IsPackageInstalled("trafficserver") { + if (serviceNeeds == t3cutil.ServiceNeedsRestart || serviceNeeds == t3cutil.ServiceNeedsReload) && !r.IsPackageInstalled(packageName) { // TODO try to reload/restart anyway? To allow non-RPM installs? - return errors.New("trafficserver needs " + serviceNeeds.String() + " but is not installed.") + return errors.New(packageName + " needs " + serviceNeeds.String() + " but is not installed.") } - svcStatus, _, err := util.GetServiceStatus("trafficserver") + svcStatus, _, err := util.GetServiceStatus(packageName) if err != nil { return errors.New("getting trafficserver service status: " + err.Error()) } @@ -1161,7 +1165,7 @@ func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cu if svcStatus != util.SvcRunning { startStr = "start" } - if _, err := util.ServiceStart("trafficserver", startStr); err != nil { + if _, err := util.ServiceStart(packageName, startStr); err != nil { t3cutil.WriteActionLog(t3cutil.ActionLogActionATSRestart, t3cutil.ActionLogStatusFailure, metaData) return errors.New("failed to restart trafficserver") } @@ -1188,7 +1192,13 @@ func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cu log.Errorln("ATS configuration has changed. The new config will be picked up the next time ATS is started.") } else if serviceNeeds == t3cutil.ServiceNeedsReload { log.Infoln("ATS configuration has changed, Running 'traffic_ctl config reload' now.") - if _, _, err := util.ExecCommand(config.TSHome+config.TrafficCtl, "config", "reload"); err != nil { + reloadCommand := config.TSHome + config.TrafficCtl + reloadArgs := []string{"config", "reload"} + if cfg.CacheType == "varnish" { + reloadCommand = "varnishreload" + reloadArgs = []string{} + } + if _, _, err := util.ExecCommand(reloadCommand, reloadArgs...); err != nil { t3cutil.WriteActionLog(t3cutil.ActionLogActionATSReload, t3cutil.ActionLogStatusFailure, metaData) if *syncdsUpdate == UpdateTropsNeeded { diff --git a/cache-config/t3c-generate/cfgfile/varnish.go b/cache-config/t3c-generate/cfgfile/varnish.go index 0f6412e73a..88763b3955 100644 --- a/cache-config/t3c-generate/cfgfile/varnish.go +++ b/cache-config/t3c-generate/cfgfile/varnish.go @@ -1,6 +1,7 @@ package cfgfile import ( + "github.com/apache/trafficcontrol/cache-config/t3c-generate/config" "github.com/apache/trafficcontrol/cache-config/t3cutil" "github.com/apache/trafficcontrol/lib/varnishcfg" ) @@ -26,10 +27,20 @@ import ( // GetVarnishConfigs returns varnish configuration files // TODO: add varnishncsa and hitch configs -func GetVarnishConfigs(toData *t3cutil.ConfigData) (string, error) { +func GetVarnishConfigs(toData *t3cutil.ConfigData, cfg config.Cfg) ([]t3cutil.ATSConfigFile, error) { vclBuilder := varnishcfg.NewVCLBuilder(toData) vcl, warnings, err := vclBuilder.BuildVCLFile() logWarnings("Generating varnish configuration files: ", warnings) - return vcl, err + configs := make([]t3cutil.ATSConfigFile, 0) + // TODO: should be parameterized and generated from varnishcfg + configs = append(configs, t3cutil.ATSConfigFile{ + Name: "default.vcl", + Text: vcl, + Path: cfg.Dir, + ContentType: "text/plain; charset=us-ascii", + LineComment: "//", + Secure: false, + }) + return configs, err } diff --git a/cache-config/t3c-generate/t3c-generate.go b/cache-config/t3c-generate/t3c-generate.go index a35e87405f..f73646df6c 100644 --- a/cache-config/t3c-generate/t3c-generate.go +++ b/cache-config/t3c-generate/t3c-generate.go @@ -86,13 +86,16 @@ func main() { } if cfg.Cache == "varnish" { - vcl, err := cfgfile.GetVarnishConfigs(toData) + configs, err := cfgfile.GetVarnishConfigs(toData, cfg) if err != nil { log.Errorln("Generating varnish config for'" + *toData.Server.HostName + "': " + err.Error()) os.Exit(config.ExitCodeErrGeneric) } - // TODO: print json for t3c-apply to consume. will be done with t3c-apply changes - fmt.Println(vcl) + err = cfgfile.WriteConfigs(configs, os.Stdout) + if err != nil { + log.Errorln("Writing configs for '" + *toData.Server.HostName + "': " + err.Error()) + os.Exit(config.ExitCodeErrGeneric) + } os.Exit(config.ExitCodeSuccess) } diff --git a/infrastructure/cdn-in-a-box/varnish/Dockerfile b/infrastructure/cdn-in-a-box/varnish/Dockerfile index 3ae0d4349e..df4d526bc2 100644 --- a/infrastructure/cdn-in-a-box/varnish/Dockerfile +++ b/infrastructure/cdn-in-a-box/varnish/Dockerfile @@ -22,35 +22,16 @@ ARG RHEL_VERSION=8 # Makes RHEL_VERSION available at runtime ENV RHEL_VERSION="$RHEL_VERSION" -RUN dnf install -y 'dnf-command(config-manager)' -RUN yum config-manager --set-enabled powertools -RUN yum install -y diffutils python3-sphinx - -RUN yum install -y epel-release && yum install -y \ - make \ - autoconf \ - automake \ - jemalloc-devel \ - libedit-devel \ - libtool \ - libunwind-devel \ - ncurses-devel \ - pcre2-devel \ - pkgconfig \ - python3-docutils \ - cpio \ - git \ - perl \ - jq \ - gettext - -RUN dnf install -y bind-utils kyotocabinet-libs initscripts iproute net-tools nmap-ncat gettext autoconf automake libtool gcc-c++ cronie glibc-devel openssl-devel && \ - dnf install -y logrotate && \ +RUN dnf module disable varnish -y && yum install -y epel-release + +RUN curl -s https://packagecloud.io/install/repositories/varnishcache/varnish73/script.rpm.sh | bash + +RUN yum install varnish-7.3.0-1.el8.x86_64 -y + +RUN dnf install -y bind-utils kyotocabinet-libs initscripts iproute net-tools nmap-ncat gettext autoconf automake libtool gcc-c++ cronie glibc-devel openssl-devel git perl && \ + dnf install -y jq logrotate findutils && \ dnf clean all -RUN curl -O https://varnish-cache.org/downloads/varnish-7.3.0.tgz && tar xf varnish-7.3.0.tgz -RUN cd varnish-7.3.0 && sh autogen.sh && sh configure && make && make install -RUN rm -rf varnish* COPY infrastructure/cdn-in-a-box/varnish/run.sh infrastructure/cdn-in-a-box/traffic_ops/to-access.sh infrastructure/cdn-in-a-box/enroller/server_template.json / @@ -59,11 +40,16 @@ COPY infrastructure/cdn-in-a-box/dns/set-dns.sh \ /usr/local/sbin/ +COPY infrastructure/cdn-in-a-box/varnish/systemctl.sh /usr/bin/systemctl + ARG ORT_RPM=infrastructure/cdn-in-a-box/cache/trafficcontrol-cache-config.rpm COPY $ORT_RPM / RUN rpm -Uvh /$(basename $ORT_RPM) &&\ rm /$(basename $ORT_RPM) +COPY infrastructure/cdn-in-a-box/varnish/traffic_ops_ort.crontab /etc/cron.d/traffic_ops_ort-cron-template + + CMD /run.sh FROM common-varnish-cache-config-layers AS mid diff --git a/infrastructure/cdn-in-a-box/varnish/run.sh b/infrastructure/cdn-in-a-box/varnish/run.sh index 2e0c2562c0..afeedb4b3f 100755 --- a/infrastructure/cdn-in-a-box/varnish/run.sh +++ b/infrastructure/cdn-in-a-box/varnish/run.sh @@ -19,6 +19,7 @@ trap 'echo "Error on line ${LINENO} of ${0}"; exit 1' ERR set -o errexit -o nounset -o pipefail -o xtrace -o monitor +env > /ciab.env mkdir /tmp/ort @@ -72,6 +73,7 @@ until [[ $(to-get "api/4.0/cdns/name/$CDN_NAME/sslkeys" | jq '.response | length echo 'waiting for SSL keys to exist' sleep 3 done +mkdir -p /tmp/trafficcontrol-cache-config # hostname is already defined in /etc/init.d/99-run.sh hostname="${hostname//-/_}" # replace - with _ @@ -79,14 +81,12 @@ hostname="${hostname^^}" # uppercase debug_variable_name="T3C_DEBUG_COMPONENT_${hostname}" debug_binary="${!debug_variable_name}" if ! type -p "$debug_binary"; then - mkdir -p /etc/varnish/ - (t3c request --get-data=config --traffic-ops-url="$TO_URL" --traffic-ops-user="$TO_USER" --traffic-ops-password="$TO_PASSWORD" | t3c generate --cache=varnish | tee /etc/varnish/default.vcl) || { echo "Failed"; } + t3c apply --cache=varnish --run-mode=badass --traffic-ops-url="$TO_URL" --traffic-ops-user="$TO_USER" --traffic-ops-password="$TO_PASSWORD" --git=yes -vv || { echo "Failed"; } fi envsubst < "/etc/cron.d/traffic_ops_ort-cron-template" > "/etc/cron.d/traffic_ops_ort-cron" && rm -f "/etc/cron.d/traffic_ops_ort-cron-template" chmod "0644" "/etc/cron.d/traffic_ops_ort-cron" && crontab "/etc/cron.d/traffic_ops_ort-cron" crond -im off -varnishd -f /etc/varnish/default.vcl varnishlog diff --git a/infrastructure/cdn-in-a-box/varnish/systemctl.sh b/infrastructure/cdn-in-a-box/varnish/systemctl.sh new file mode 100755 index 0000000000..e8b849302d --- /dev/null +++ b/infrastructure/cdn-in-a-box/varnish/systemctl.sh @@ -0,0 +1,94 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +VARNISHD_EXECUTABLE="/usr/sbin/varnishd" + +is_varnishd_running() { + pgrep -x "$(basename "$VARNISHD_EXECUTABLE")" >/dev/null +} + +start_varnishd() { + if is_varnishd_running; then + echo "varnishd is already running." + else + echo "Starting varnishd..." + "$VARNISHD_EXECUTABLE" -f /opt/trafficserver/etc/trafficserver/default.vcl + echo "varnishd is now running." + fi +} + +stop_varnishd() { + if is_varnishd_running; then + echo "Stopping varnishd..." + + # Send SIGTERM signal to varnishd to terminate gracefully + pkill -x "$(basename "$VARNISHD_EXECUTABLE")" + + # Wait for varnishd to stop, giving it a timeout of 10 seconds + timeout=10 + while is_varnishd_running; do + if ((timeout-- == 0)); then + echo "Timed out waiting for varnishd to stop. Sending SIGKILL..." + pkill -9 -x "$(basename "$VARNISHD_EXECUTABLE")" + break + fi + sleep 1 + done + + if is_varnishd_running; then + echo "Failed to stop varnishd." + else + echo "varnishd is stopped." + fi + else + echo "varnishd is not running." + fi +} + +restart_varnishd() { + echo "Restarting varnishd..." + stop_varnishd + start_varnishd +} + +case "$1" in + enable) + exit 0 + ;; + start) + start_varnishd + ;; + stop) + stop_varnishd + ;; + restart) + restart_varnishd + ;; + status) + if is_varnishd_running; then + exit 0 + fi + exit 3 + ;; + *) + echo "Usage: $0 {start|stop|restart|enable|status}" + exit 1 +esac + +exit 0 diff --git a/infrastructure/cdn-in-a-box/varnish/traffic_ops_ort.crontab b/infrastructure/cdn-in-a-box/varnish/traffic_ops_ort.crontab new file mode 100644 index 0000000000..d830ed0062 --- /dev/null +++ b/infrastructure/cdn-in-a-box/varnish/traffic_ops_ort.crontab @@ -0,0 +1,18 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +*/1 * * * * root t3c apply --cache=varnish --run-mode=syncds --traffic-ops-url=$TO_URL --traffic-ops-user=$TO_USER --traffic-ops-password=$TO_PASSWORD --git=yes -vv --cache-host-name=$(hostname -s) >> /var/log/ort.log 2>&1 diff --git a/lib/varnishcfg/vcl.go b/lib/varnishcfg/vcl.go index 02215eb171..a0f777ac9e 100644 --- a/lib/varnishcfg/vcl.go +++ b/lib/varnishcfg/vcl.go @@ -55,7 +55,7 @@ func (v vclFile) String() string { } // varnishd will fail if there are no backends defined if len(v.backends) == 0 { - txt += fmt.Sprint("backend default none;") + txt += fmt.Sprint("backend default none;\n") } for name, acl := range v.acls {