Add parents to varnish cache (#7669)

* Add varnishcfg package and parent configuration

* Add Varnish Dockerfile to be used in CIAB

* Add license to Varnsih Dockerfile

* Add systemctl.sh to handle Varnish service and integrate Varnish with t3c-apply

* Move host changes to BE fetch, change varnish dir and make test more readable

* Remove Varnish package release and arch, add GoDoc and move licenses

* Move license text

cache-config/t3c-apply/config/config.go

@@ -123,6 +123,7 @@ type Cfg struct {
 	Version           string
 	GitRevision       string
 	LocalATSVersion   string
+	CacheType         string
 }
 
 func (cfg Cfg) AppVersion() string { return t3cutil.VersionStr(AppName, cfg.Version, cfg.GitRevision) }
@@ -277,6 +278,7 @@ func GetCfg(appVersion string, gitRevision string) (Cfg, error) {
 	defaultClientTLSVersions := getopt.StringLong("default-client-tls-versions", 'V', "", "Comma-delimited list of default TLS versions for Delivery Services with no Parameter, e.g. --default-tls-versions='1.1,1.2,1.3'. If omitted, all versions are enabled.")
 	maxmindLocationPtr := getopt.StringLong("maxmind-location", 'M', "", "URL of a maxmind gzipped database file, to be installed into the trafficserver etc directory.")
 	verbosePtr := getopt.CounterLong("verbose", 'v', `Log verbosity. Logging is output to stderr. By default, errors are logged. To log warnings, pass '-v'. To log info, pass '-vv'. To omit error logging, see '-s'`)
+	cache := getopt.StringLong("cache", 'T', "ats", "Cache server type. Generate configuration files for specific cache server type, e.g. 'ats', 'varnish'.")
 	const silentFlagName = "silent"
 	silentPtr := getopt.BoolLong(silentFlagName, 's', `Silent. Errors are not logged, and the 'verbose' flag is ignored. If a fatal error occurs, the return code will be non-zero but no text will be output to stderr`)
 
@@ -533,6 +535,9 @@ If any of the related flags are also set, they override the mode's default behav
 	if tsHome != "" {
 		TSHome = tsHome
 		tsConfigDir = tsHome + "/etc/trafficserver"
+		if cache != nil && *cache == "varnish" {
+			tsConfigDir = tsHome + "/etc/varnish"
+		}
 		toInfoLog = append(toInfoLog, fmt.Sprintf("TSHome: %s, TSConfigDir: %s\n", TSHome, tsConfigDir))
 	}
 
@@ -612,6 +617,7 @@ If any of the related flags are also set, they override the mode's default behav
 		Version:                     appVersion,
 		GitRevision:                 gitRevision,
 		LocalATSVersion:             atsVersionStr,
+		CacheType:                   *cache,
 	}
 
 	if err = log.InitCfg(cfg); err != nil {

cache-config/t3c-apply/t3c-apply.go

@@ -220,7 +220,7 @@ func Main() int {
 		}
 
 	} else {
-		syncdsUpdate, err = trops.CheckSyncDSState(metaData)
+		syncdsUpdate, err = trops.CheckSyncDSState(metaData, cfg)
 		if err != nil {
 			log.Errorln("Checking syncds state: " + err.Error())
 			return GitCommitAndExit(ExitCodeSyncDSError, FailureExitMsg, cfg, metaData, oldMetaData)
@@ -241,7 +241,7 @@ func Main() int {
 				} else if rc == 0 {
 					log.Infoln("updated the remap.config for reloading.")
 				}
-				if err := trops.StartServices(&syncdsUpdate, metaData); err != nil {
+				if err := trops.StartServices(&syncdsUpdate, metaData, cfg); err != nil {
 					log.Errorln("failed to start services: " + err.Error())
 					metaData.PartialSuccess = true
 					return GitCommitAndExit(ExitCodeServicesError, PostConfigFailureExitMsg, cfg, metaData, oldMetaData)
@@ -311,7 +311,7 @@ func Main() int {
 		}
 	}
 
-	if err := trops.StartServices(&syncdsUpdate, metaData); err != nil {
+	if err := trops.StartServices(&syncdsUpdate, metaData, cfg); err != nil {
 		log.Errorln("failed to start services: " + err.Error())
 		metaData.PartialSuccess = true
 		return GitCommitAndExit(ExitCodeServicesError, PostConfigFailureExitMsg, cfg, metaData, oldMetaData)
@@ -377,7 +377,7 @@ func GitCommitAndExit(exitCode int, exitMsg string, cfg config.Cfg, metaData *t3
 	// so add the old files to the new metadata.
 	// This is especially important for reval runs, which don't add all files.
 	metaData.OwnedFilePaths = t3cutil.CombineOwnedFilePaths(metaData, oldMetaData)
-	if len(metaData.InstalledPackages) == 0 {
+	if len(metaData.InstalledPackages) == 0 && oldMetaData != nil {
 		metaData.InstalledPackages = oldMetaData.InstalledPackages
 	}
 	WriteMetaData(cfg, metaData)

cache-config/t3c-apply/torequest/cmd.go

@@ -72,6 +72,7 @@ func generate(cfg config.Cfg) ([]t3cutil.ATSConfigFile, error) {
 	args := []string{
 		`generate`,
 		"--dir=" + cfg.TsConfigDir,
+		"--cache=" + cfg.CacheType,
 	}
 
 	if cfg.LogLocationErr == log.LogLocationNull {

cache-config/t3c-apply/torequest/torequest.go

@@ -748,7 +748,7 @@ func (r *TrafficOpsReq) CheckRevalidateState(sleepOverride bool) (UpdateStatus,
 
 // CheckSyncDSState retrieves and returns the DS Update status from Traffic Ops.
 // The metaData is this run's metadata. It must not be nil, and this function may add to it.
-func (r *TrafficOpsReq) CheckSyncDSState(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) {
+func (r *TrafficOpsReq) CheckSyncDSState(metaData *t3cutil.ApplyMetaData, cfg config.Cfg) (UpdateStatus, error) {
 	updateStatus := UpdateTropsNotNeeded
 	randDispSec := time.Duration(0)
 	log.Debugln("Checking syncds state.")
@@ -785,7 +785,7 @@ func (r *TrafficOpsReq) CheckSyncDSState(metaData *t3cutil.ApplyMetaData) (Updat
 			}
 		} else if !r.Cfg.IgnoreUpdateFlag {
 			log.Errorln("no queued update needs to be applied.  Running revalidation before exiting.")
-			r.RevalidateWhileSleeping(metaData)
+			r.RevalidateWhileSleeping(metaData, cfg)
 			return UpdateTropsNotNeeded, nil
 		} else {
 			log.Errorln("Traffic Ops is signaling that no update is waiting to be applied.")
@@ -1091,7 +1091,7 @@ func (r *TrafficOpsReq) ProcessPackagesWithMetaData(packageMetaData []string) er
 	return nil
 }
 
-func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData) (UpdateStatus, error) {
+func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData, cfg config.Cfg) (UpdateStatus, error) {
 	updateStatus, err := r.CheckRevalidateState(true)
 	if err != nil {
 		return updateStatus, err
@@ -1115,7 +1115,7 @@ func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData)
 			t3cutil.WriteActionLog(t3cutil.ActionLogActionUpdateFilesReval, t3cutil.ActionLogStatusSuccess, metaData)
 		}
 
-		if err := r.StartServices(&updateStatus, metaData); err != nil {
+		if err := r.StartServices(&updateStatus, metaData, cfg); err != nil {
 			return updateStatus, errors.New("failed to start services: " + err.Error())
 		}
 
@@ -1132,7 +1132,7 @@ func (r *TrafficOpsReq) RevalidateWhileSleeping(metaData *t3cutil.ApplyMetaData)
 // StartServices reloads, restarts, or starts ATS as necessary,
 // according to the changed config files and run mode.
 // Returns nil on success or any error.
-func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cutil.ApplyMetaData) error {
+func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cutil.ApplyMetaData, cfg config.Cfg) error {
 	serviceNeeds := t3cutil.ServiceNeedsNothing
 	if r.Cfg.ServiceAction == t3cutil.ApplyServiceActionFlagRestart {
 		serviceNeeds = t3cutil.ServiceNeedsRestart
@@ -1154,13 +1154,17 @@ func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cu
 			serviceNeeds = t3cutil.ServiceNeedsReload
 		}
 	}
+	packageName := "trafficserver"
+	if cfg.CacheType == "varnish" {
+		packageName = "varnish"
+	}
 
-	if (serviceNeeds == t3cutil.ServiceNeedsRestart || serviceNeeds == t3cutil.ServiceNeedsReload) && !r.IsPackageInstalled("trafficserver") {
+	if (serviceNeeds == t3cutil.ServiceNeedsRestart || serviceNeeds == t3cutil.ServiceNeedsReload) && !r.IsPackageInstalled(packageName) {
 		// TODO try to reload/restart anyway? To allow non-RPM installs?
-		return errors.New("trafficserver needs " + serviceNeeds.String() + " but is not installed.")
+		return errors.New(packageName + " needs " + serviceNeeds.String() + " but is not installed.")
 	}
 
-	svcStatus, _, err := util.GetServiceStatus("trafficserver")
+	svcStatus, _, err := util.GetServiceStatus(packageName)
 	if err != nil {
 		return errors.New("getting trafficserver service status: " + err.Error())
 	}
@@ -1177,7 +1181,7 @@ func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cu
 		if svcStatus != util.SvcRunning {
 			startStr = "start"
 		}
-		if _, err := util.ServiceStart("trafficserver", startStr); err != nil {
+		if _, err := util.ServiceStart(packageName, startStr); err != nil {
 			t3cutil.WriteActionLog(t3cutil.ActionLogActionATSRestart, t3cutil.ActionLogStatusFailure, metaData)
 			return errors.New("failed to restart trafficserver")
 		}
@@ -1204,7 +1208,13 @@ func (r *TrafficOpsReq) StartServices(syncdsUpdate *UpdateStatus, metaData *t3cu
 			log.Errorln("ATS configuration has changed.  The new config will be picked up the next time ATS is started.")
 		} else if serviceNeeds == t3cutil.ServiceNeedsReload {
 			log.Infoln("ATS configuration has changed, Running 'traffic_ctl config reload' now.")
-			if _, _, err := util.ExecCommand(config.TSHome+config.TrafficCtl, "config", "reload"); err != nil {
+			reloadCommand := config.TSHome + config.TrafficCtl
+			reloadArgs := []string{"config", "reload"}
+			if cfg.CacheType == "varnish" {
+				reloadCommand = "varnishreload"
+				reloadArgs = []string{}
+			}
+			if _, _, err := util.ExecCommand(reloadCommand, reloadArgs...); err != nil {
 				t3cutil.WriteActionLog(t3cutil.ActionLogActionATSReload, t3cutil.ActionLogStatusFailure, metaData)
 
 				if *syncdsUpdate == UpdateTropsNeeded {

cache-config/t3c-generate/cfgfile/varnish.go

@@ -0,0 +1,46 @@
+package cfgfile
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import (
+	"github.com/apache/trafficcontrol/cache-config/t3c-generate/config"
+	"github.com/apache/trafficcontrol/cache-config/t3cutil"
+	"github.com/apache/trafficcontrol/lib/varnishcfg"
+)
+
+// GetVarnishConfigs returns varnish configuration files
+// TODO: add varnishncsa and hitch configs
+func GetVarnishConfigs(toData *t3cutil.ConfigData, cfg config.Cfg) ([]t3cutil.ATSConfigFile, error) {
+	vclBuilder := varnishcfg.NewVCLBuilder(toData)
+	vcl, warnings, err := vclBuilder.BuildVCLFile()
+	logWarnings("Generating varnish configuration files: ", warnings)
+
+	configs := make([]t3cutil.ATSConfigFile, 0)
+	// TODO: should be parameterized and generated from varnishcfg
+	configs = append(configs, t3cutil.ATSConfigFile{
+		Name:        "default.vcl",
+		Text:        vcl,
+		Path:        cfg.Dir,
+		ContentType: "text/plain; charset=us-ascii",
+		LineComment: "//",
+		Secure:      false,
+	})
+	return configs, err
+}

cache-config/t3c-generate/config/config.go

@@ -62,6 +62,7 @@ type Cfg struct {
 	DefaultTLSVersions []atscfg.TLSVersion
 	Version            string
 	GitRevision        string
+	Cache              string
 }
 
 func (cfg Cfg) ErrorLog() log.LogLocation   { return log.LogLocation(cfg.LogLocationErr) }
@@ -88,6 +89,7 @@ func GetCfg(appVersion string, gitRevision string) (Cfg, error) {
 	atsVersion := getopt.StringLong("ats-version", 'a', "", "The ATS version, e.g. 9.1.2-42.abc123.el7.x86_64. If omitted, generation will attempt to get the ATS version from the Server Parameters, and fall back to lib/go-atscfg.DefaultATSVersion")
 	verbosePtr := getopt.CounterLong("verbose", 'v', `Log verbosity. Logging is output to stderr. By default, errors are logged. To log warnings, pass '-v'. To log info, pass '-vv'. To omit error logging, see '-s'`)
 	silentPtr := getopt.BoolLong("silent", 's', `Silent. Errors are not logged, and the 'verbose' flag is ignored. If a fatal error occurs, the return code will be non-zero but no text will be output to stderr`)
+	cache := getopt.StringLong("cache", 'C', "ats", "Cache server type. Generate configuration files for specific cache server type, e.g. 'ats', 'varnish'.")
 
 	const useStrategiesFlagName = "use-strategies"
 	const defaultUseStrategies = t3cutil.UseStrategiesFlagFalse
@@ -185,6 +187,7 @@ func GetCfg(appVersion string, gitRevision string) (Cfg, error) {
 		GitRevision:        gitRevision,
 		UseStrategies:      t3cutil.UseStrategiesFlag(*useStrategiesPtr),
 		GoDirect:           *goDirectPtr,
+		Cache:              *cache,
 	}
 	if err := log.InitCfg(cfg); err != nil {
 		return Cfg{}, errors.New("Initializing loggers: " + err.Error() + "\n")

cache-config/t3c-generate/t3c-generate.go

@@ -85,6 +85,20 @@ func main() {
 		os.Exit(config.ExitCodeErrGeneric)
 	}
 
+	if cfg.Cache == "varnish" {
+		configs, err := cfgfile.GetVarnishConfigs(toData, cfg)
+		if err != nil {
+			log.Errorln("Generating varnish config for'" + *toData.Server.HostName + "': " + err.Error())
+			os.Exit(config.ExitCodeErrGeneric)
+		}
+		err = cfgfile.WriteConfigs(configs, os.Stdout)
+		if err != nil {
+			log.Errorln("Writing configs for '" + *toData.Server.HostName + "': " + err.Error())
+			os.Exit(config.ExitCodeErrGeneric)
+		}
+		os.Exit(config.ExitCodeSuccess)
+	}
+
 	configs, err := cfgfile.GetAllConfigs(toData, cfg)
 	if err != nil {
 		log.Errorln("Getting config for'" + *toData.Server.HostName + "': " + err.Error())

infrastructure/cdn-in-a-box/enroller/Dockerfile

@@ -42,6 +42,9 @@ COPY ./traffic_ops/toclientlib/ /go/src/github.com/apache/trafficcontrol/traffic
 COPY ./traffic_ops/v4-client/ /go/src/github.com/apache/trafficcontrol/traffic_ops/v4-client/
 COPY ./infrastructure/cdn-in-a-box/ /go/src/github.com/apache/trafficcontrol/infrastructure/cdn-in-a-box/
 
+# varnishcfg requires t3c for ToData struct and not needed for enroller
+RUN rm -rf /go/src/github.com/apache/trafficcontrol/lib/varnishcfg
+
 WORKDIR /go/src/github.com/apache/trafficcontrol/infrastructure/cdn-in-a-box/enroller
 RUN set -o errexit -o nounset; \
     go clean; \

infrastructure/cdn-in-a-box/varnish/Dockerfile

@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+ARG BASE_IMAGE=rockylinux \
+    RHEL_VERSION=8
+FROM ${BASE_IMAGE}:${RHEL_VERSION} AS common-varnish-cache-config-layers
+ARG RHEL_VERSION=8
+# Makes RHEL_VERSION available at runtime
+ENV RHEL_VERSION="$RHEL_VERSION"
+
+RUN dnf module disable varnish -y && yum install -y epel-release
+
+RUN curl -s https://packagecloud.io/install/repositories/varnishcache/varnish73/script.rpm.sh | bash
+
+RUN yum install varnish-7.3.0 -y
+
+RUN dnf install -y bind-utils kyotocabinet-libs initscripts iproute net-tools nmap-ncat gettext autoconf automake libtool gcc-c++ cronie glibc-devel openssl-devel git perl && \
+    dnf install -y jq logrotate findutils && \
+    dnf clean all
+
+
+COPY infrastructure/cdn-in-a-box/varnish/run.sh infrastructure/cdn-in-a-box/traffic_ops/to-access.sh infrastructure/cdn-in-a-box/enroller/server_template.json /
+
+COPY infrastructure/cdn-in-a-box/dns/set-dns.sh \
+    infrastructure/cdn-in-a-box/dns/insert-self-into-dns.sh \
+    /usr/local/sbin/
+
+
+COPY infrastructure/cdn-in-a-box/varnish/systemctl.sh /usr/bin/systemctl
+
+ARG ORT_RPM=infrastructure/cdn-in-a-box/cache/trafficcontrol-cache-config.rpm
+COPY $ORT_RPM /
+RUN rpm -Uvh /$(basename $ORT_RPM) &&\
+    rm /$(basename $ORT_RPM)
+
+COPY infrastructure/cdn-in-a-box/varnish/traffic_ops_ort.crontab /etc/cron.d/traffic_ops_ort-cron-template
+
+
+CMD /run.sh
+
+FROM common-varnish-cache-config-layers AS mid
+ENV CACHE_TYPE=mid
+COPY infrastructure/cdn-in-a-box/mid/init.d/ /opt/init.d/
+
+FROM common-varnish-cache-config-layers AS edge
+ENV CACHE_TYPE=edge
+COPY infrastructure/cdn-in-a-box/edge/init.d/ /opt/init.d/
+
+