Internal: Set image projects via configs

dev-docs/new-distro.md

@@ -8,10 +8,10 @@ the rest of the steps are documented in a Google-internal markdown doc.
 The following steps refer to `$DISTRO_SHORT` and `$DISTRO_FAMILY`. You will
 need to replace these with appropriate names for your new distro. Example
 values for these variables are: `hirsute` for `$DISTRO_SHORT` and
-`ubuntu-2104` for `$DISTRO_FAMILY`.
+`ubuntu-os-cloud:ubuntu-2104` for `$DISTRO_FAMILY`.
 
 NOTE: Some Ubuntu releases are LTS and some are not. For LTS releases, make
-sure the distro family ends in `-lts`. For example, `ubuntu-2004-lts`.
+sure the distro family ends in `-lts`. For example, `ubuntu-os-cloud:ubuntu-2004-lts`.
 
 For a list of existing distro families, consult
 [image_lists.gcl](https://github.com/GoogleCloudPlatform/ops-agent/blob/master/kokoro/config/test/image_lists.gcl)
@@ -46,7 +46,7 @@ manifest as runtime errors that won't show up until tests are run.
     [kokoro/config/test/ops_agent/presubmit/bullseye_x86_64.gcl](https://github.com/GoogleCloudPlatform/ops-agent/blob/master/kokoro/config/test/ops_agent/presubmit/bullseye_x86_64.gcl)
     or [kokoro/config/test/ops_agent/presubmit/bullseye_aarch64.gcl](https://github.com/GoogleCloudPlatform/ops-agent/blob/master/kokoro/config/test/ops_agent/presubmit/bullseye_aarch64.gcl),
     replace `image_lists.bullseye_x86_64.presubmit` with
-    `$DISTRO_FAMILY` (in a single-element list, like `['sles-15']`).
+    `$DISTRO_FAMILY` (in a single-element list, like `['suse-cloud:sles-15']`).
     This will cause the bullseye Kokoro build to build and test your
     distro instead.
 
@@ -100,7 +100,7 @@ The instructions are very similar to the instructions for `ops_agent_test`.
         or
         [test/third_party_apps/presubmit/bullseye_aarch64.gcl](https://github.com/GoogleCloudPlatform/ops-agent/blob/master/kokoro/config/test/third_party_apps/bullseye_aarch64.gcl),
         set `platforms` to `['$DISTRO_FAMILY']` (instead of
-        `['debian-11']`). You should also add this section right after the
+        `['debian-cloud:debian-11']`). You should also add this section right after the
         `platforms =` line:
 
         ```gcl

integration_test/README.md

@@ -50,7 +50,7 @@ Alternatively, you can export `PROJECT` and `TRANSFERS_BUCKET` in your
 environment and simply call the target.  
 You can also specify the `ZONES` and `PLATFORMS` variables if you would like
 to run the tests on something other than the defaults (`us-central1-b` for
-ZONES and `debian-11` for `PLATFORMS`).
+ZONES and `debian-cloud:debian-11` for `PLATFORMS`).
 
 The above command will run the tests against the stable Ops Agent. To test
 against a pre-built but unreleased agent, you can use add the

integration_test/agents/agents.go

@@ -866,7 +866,7 @@ func CommonSetupWithExtraCreateArgumentsAndMetadata(t *testing.T, platform strin
 	logger := gce.SetupLogger(t)
 	logger.ToMainLog().Println("Calling SetupVM(). For details, see VM_initialization.txt.")
 	options := gce.VMOptions{
-		Platform:             platform,
+		ImageSpec:            platform,
 		TimeToLive:           "3h",
 		MachineType:          RecommendedMachineType(platform),
 		ExtraCreateArguments: extraCreateArguments,

integration_test/gce/gce_testing.go

@@ -36,7 +36,7 @@ AGENT_PACKAGES_IN_GCS, for details see README.md.
 
 	PROJECT=dev_project \
 	ZONES=us-central1-b \
-	PLATFORMS=debian-10,centos-8,rhel-8-2-sap-ha,sles-15,ubuntu-2004-lts,windows-2016,windows-2019 \
+	PLATFORMS=debian-cloud:debian-10,rocky-linux-cloud:rocky-linux-8,rhel-sap-cloud:rhel-8-2-sap-ha,suse-cloud:sles-15,ubuntu-os-cloud:ubuntu-2004-lts,windows-cloud:windows-2016,windows-cloud:windows-2019 \
 	go test -v ops_agent_test.go \
 	  -test.parallel=1000 \
 	  -tags=integration_test \
@@ -291,6 +291,8 @@ type VM struct {
 	Project     string
 	Network     string
 	Platform    string
+	// The VMOptions.ImageSpec used to create the VM.
+	ImageSpec   string
 	Zone        string
 	MachineType string
 	ID          int64
@@ -301,43 +303,6 @@ type VM struct {
 	AlreadyDeleted bool
 }
 
-// imageProject returns the image project providing the given image family.
-func imageProject(family string) (string, error) {
-	firstWord := strings.Split(family, "-")[0]
-	switch firstWord {
-	case "windows":
-		return "windows-cloud", nil
-	case "sql":
-		return "windows-sql-cloud", nil
-	case "centos":
-		return "centos-cloud", nil
-	case "debian":
-		return "debian-cloud", nil
-	case "ubuntu":
-		return "ubuntu-os-cloud", nil
-	case "rhel":
-		// There are a few different cases:
-		// "rhel-7", "rhel-7-4-sap", and "rhel-7-6-sap-ha".
-		if strings.Contains(family, "-sap") {
-			return "rhel-sap-cloud", nil
-		}
-		return "rhel-cloud", nil
-	case "rocky":
-		return "rocky-linux-cloud", nil
-	case "opensuse":
-		return "opensuse-cloud", nil
-	case "sles":
-		// There are a few different cases:
-		// "sles-15" and "sles-15-sp*-sap".
-		if strings.Contains(family, "-sap") {
-			return "suse-sap-cloud", nil
-		}
-		return "suse-cloud", nil
-	default:
-		return "", fmt.Errorf("could not find match for family %s", family)
-	}
-}
-
 // SyslogLocation returns a filesystem path to the system log. This function
 // assumes the platform is some kind of Linux.
 func SyslogLocation(platform string) string {
@@ -1016,12 +981,6 @@ func prepareSLES(ctx context.Context, logger *log.Logger, vm *VM) error {
 	return err
 }
 
-var (
-	overriddenImageFamilies = map[string]string{
-		"opensuse-leap-15-4": "opensuse-leap-15-4-v20231208-x86-64",
-	}
-)
-
 func addFrameworkMetadata(platform string, inputMetadata map[string]string) (map[string]string, error) {
 	metadataCopy := make(map[string]string)
 
@@ -1099,21 +1058,73 @@ func getVMPlatform(image string, platform string) (string, error) {
 	return "", errors.New("at least one of image or platform must be specified")
 }
 
+// In cases where ImageSpec is not being used yet, construct it from known fields.
+func constructImageSpec(options *VMOptions) {
+	if options.ImageSpec != "" || options.ImageProject == "" {
+		return
+	}
+	if options.Platform != "" {
+		options.ImageSpec = fmt.Sprintf("%s:%s", options.ImageProject, options.Platform)
+	} else if options.Image != "" {
+		options.ImageSpec = fmt.Sprintf("%s=%s", options.ImageProject, options.Image)
+	}
+}
+
+// parseImageSpec looks for the ImageSpec field in VMOptions and sets
+// ImageProject/Image/Platform accordingly.
+func parseImageSpec(options *VMOptions) (error) {
+	if options.ImageSpec == "" {
+		constructImageSpec(options)
+		return nil
+	}
+
+	if options.Image != "" || options.ImageProject != "" || options.Platform != "" {
+		return fmt.Errorf("If options.ImageSpec is set, options.(Image|ImageProject|Platform) cannot be: %+v", options)
+	}
+
+	delim := ""
+	if strings.Contains(options.ImageSpec, ":"){
+		delim = ":"
+	} else if strings.Contains(options.ImageSpec, "="){
+		delim = "="
+	} else {
+		return fmt.Errorf("could not parse options.ImageSpec from struct: %+v", options)
+	}
+
+	s := strings.Split(options.ImageSpec, delim)
+	options.ImageProject = s[0]
+
+	switch delim  {
+		case ":":
+			options.Platform = s[1]
+		case "=":
+			options.Image = s[1]
+	}
+
+	return nil
+}
+
 // attemptCreateInstance creates a VM instance and waits for it to be ready.
 // Returns a VM object or an error (never both). The caller is responsible for
 // deleting the VM if (and only if) the returned error is nil.
 func attemptCreateInstance(ctx context.Context, logger *log.Logger, options VMOptions) (vmToReturn *VM, errToReturn error) {
 
+	err := parseImageSpec(&options)
+	if err != nil {
+		return nil, err
+	}
+
 	platform, err := getVMPlatform(options.Image, options.Platform)
 	if err != nil {
 		return nil, err
 	}
 	vm := &VM{
-		Project:  options.Project,
-		Platform: platform,
-		Name:     options.Name,
-		Network:  os.Getenv("NETWORK_NAME"),
-		Zone:     options.Zone,
+		Project:   options.Project,
+		Platform:  platform,
+		ImageSpec: options.ImageSpec,
+		Name:      options.Name,
+		Network:   os.Getenv("NETWORK_NAME"),
+		Zone:      options.Zone,
 	}
 	if vm.Name == "" {
 		// The VM name needs to adhere to these restrictions:
@@ -1144,13 +1155,6 @@ func attemptCreateInstance(ctx context.Context, logger *log.Logger, options VMOp
 	}
 
 	imgProject := options.ImageProject
-	if imgProject == "" {
-		var err error
-		imgProject, err = imageProject(vm.Platform)
-		if err != nil {
-			return nil, fmt.Errorf("attemptCreateInstance() could not find image project: %v", err)
-		}
-	}
 	newMetadata, err := addFrameworkMetadata(vm.Platform, options.Metadata)
 	if err != nil {
 		return nil, fmt.Errorf("attemptCreateInstance() could not construct valid metadata: %v", err)
@@ -1164,11 +1168,6 @@ func attemptCreateInstance(ctx context.Context, logger *log.Logger, options VMOp
 
 	if options.Platform != "" {
 		imageOrImageFamilyFlag = "--image-family=" + options.Platform
-
-		if image, ok := overriddenImageFamilies[options.Platform]; ok {
-			imageOrImageFamilyFlag = "--image=" + image
-		}
-
 	}
 
 	imageFamilyScope := options.ImageFamilyScope
@@ -1870,6 +1869,13 @@ func SetupLogger(t *testing.T) *logging.DirectoryLogger {
 
 // VMOptions specifies settings when creating a VM via CreateInstance() or SetupVM().
 type VMOptions struct {
+	// Optional. Can be used to pass image/image family & image project in one
+	// string. If set, Platform/Image/ImageProject should not be set.
+	//
+	// Example Image Specs:
+	// Image Family / Project: `<project>:<family>`
+	// Specific Image / Project: `<project>=<image>``
+	ImageSpec string
 	// Required. Normally passed as --image-family to
 	// "gcloud compute images create".
 	Platform string

integration_test/soak_test/README.md

@@ -6,7 +6,7 @@ parameters as desired) and run:
 
 ```
 PROJECT=my_project \
-  DISTRO=debian-11 \
+  DISTRO=debian-cloud:debian-11 \
   ZONES=us-central1-b \
   TTL=100m \
   LOG_SIZE_IN_BYTES=1000 \

integration_test/soak_test/cmd/launcher/main.go

@@ -29,7 +29,7 @@ LOG_SIZE_IN_BYTES: How many bytes each log entry should be.
 
 TTL: How long to keep the VM alive, expressed as "24h30m" or similar.
 
-DISTRO: The GCE image family name to run, e.g. "debian-11".
+DISTRO: The Image Spec to run the tests on, e.g. "debian-cloud:debian-11".
 
 VM_NAME: (Optional) The name of the VM to spawn. If not supplied,
 a random name will be generated by gce_testing.go.
@@ -122,11 +122,11 @@ func mainErr() error {
 
 	// Create the VM.
 	options := gce.VMOptions{
-		Platform:    distro,
-		TimeToLive:  ttl,
-		Name:        vmName,
-		MachineType: "e2-standard-16",
-		Metadata: map[string]string{
+		ImageSpec:    distro,
+		TimeToLive:   ttl,
+		Name:         vmName,
+		MachineType:  "e2-standard-16",
+		Metadata:     map[string]string{
 			// This is to avoid Windows updates and reboots (b/295165549), and
 			// also to avoid throughput blips when the OS Config agent runs
 			// periodically.

integration_test/third_party_apps_data/applications/aerospike/metadata.yaml

@@ -131,11 +131,11 @@ minimum_supported_agent_version:
 supported_operating_systems: linux
 platforms_to_skip:
   # Unable to install Aerospike on various distros.
-  - debian-12
-  - debian-12-arm64
-  - sles-12
-  - rocky-linux-9
-  - rocky-linux-9-arm64
-  - ubuntu-2310-amd64
-  - ubuntu-2310-arm64
+  - debian-cloud:debian-12
+  - debian-cloud:debian-12-arm64
+  - suse-cloud:sles-12
+  - rocky-linux-cloud:rocky-linux-9
+  - rocky-linux-cloud:rocky-linux-9-arm64
+  - ubuntu-os-cloud:ubuntu-2310-amd64
+  - ubuntu-os-cloud:ubuntu-2310-arm64
 public_url: https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/third-party/aerospike

integration_test/third_party_apps_data/applications/apache/metadata.yaml

@@ -61,10 +61,10 @@ expected_metrics:
       server_name: .*
     unavailable_on:
       # https://github.com/GoogleCloudPlatform/ops-agent/issues/1173
-      - centos-7
-      - sles-12
-      - sles-15
-      - sles-15-arm64
+      - centos-cloud:centos-7
+      - suse-cloud:sles-12
+      - suse-cloud:sles-15
+      - suse-cloud:sles-15-arm64
   - type: workload.googleapis.com/apache.requests
     value_type: INT64
     kind: CUMULATIVE
@@ -171,10 +171,10 @@ expected_logs:
           # Unavailable for the same reasons as current_connections,
           # described in:
           # https://github.com/GoogleCloudPlatform/ops-agent/issues/1173
-          - centos-7
-          - sles-12
-          - sles-15
-          - sles-15-arm64
+          - centos-cloud:centos-7
+          - suse-cloud:sles-12
+          - suse-cloud:sles-15
+          - suse-cloud:sles-15-arm64
       - name: severity
         type: string
         description: ''

integration_test/third_party_apps_data/applications/cassandra/metadata.yaml

@@ -48,14 +48,14 @@ minimum_supported_agent_version:
 supported_operating_systems: linux
 # TODO: Fix errors and enable tests on all platforms.
 platforms_to_skip:
-  - centos-7 # see https://issues.apache.org/jira/browse/CASSANDRA-18059 for fix.
-  - sles-12 # QueryLog() failed: cassandra_system not found, exhausted retries; QueryLog() failed: cassandra_debug not found, exhausted retries; QueryLog() failed: cassandra_gc not found, exhausted retries
-  - sles-15 # QueryLog() failed: cassandra_system not found, exhausted retries; QueryLog() failed: cassandra_debug not found, exhausted retries; QueryLog() failed: cassandra_gc not found, exhausted retries
-  - sles-15-arm64
-  - debian-12 # QueryLog() failed: cassandra_system not found, exhausted retries; QueryLog() failed: cassandra_debug not found, exhausted retries; QueryLog() failed: cassandra_gc not found, exhausted retries
-  - debian-12-arm64 # QueryLog() failed: cassandra_system not found, exhausted retries; QueryLog() failed: cassandra_debug not found, exhausted retries; QueryLog() failed: cassandra_gc not found, exhausted retries
-  - ubuntu-2004-lts # GPG error [...] the public key is not available: NO_PUBKEY AA8E81B4331F7F50 NO_PUBKEY 112695A0E562B32A
-  - ubuntu-2204-lts # GPG error [...] the public key is not available: NO_PUBKEY AA8E81B4331F7F50 NO_PUBKEY 112695A0E562B32A
+  - centos-cloud:centos-7 # see https://issues.apache.org/jira/browse/CASSANDRA-18059 for fix.
+  - suse-cloud:sles-12 # QueryLog() failed: cassandra_system not found, exhausted retries; QueryLog() failed: cassandra_debug not found, exhausted retries; QueryLog() failed: cassandra_gc not found, exhausted retries
+  - suse-cloud:sles-15 # QueryLog() failed: cassandra_system not found, exhausted retries; QueryLog() failed: cassandra_debug not found, exhausted retries; QueryLog() failed: cassandra_gc not found, exhausted retries
+  - suse-cloud:sles-15-arm64
+  - debian-cloud:debian-12 # QueryLog() failed: cassandra_system not found, exhausted retries; QueryLog() failed: cassandra_debug not found, exhausted retries; QueryLog() failed: cassandra_gc not found, exhausted retries
+  - debian-cloud:debian-12-arm64 # QueryLog() failed: cassandra_system not found, exhausted retries; QueryLog() failed: cassandra_debug not found, exhausted retries; QueryLog() failed: cassandra_gc not found, exhausted retries
+  - ubuntu-os-cloud:ubuntu-2004-lts # GPG error [...] the public key is not available: NO_PUBKEY AA8E81B4331F7F50 NO_PUBKEY 112695A0E562B32A
+  - ubuntu-os-cloud:ubuntu-2204-lts # GPG error [...] the public key is not available: NO_PUBKEY AA8E81B4331F7F50 NO_PUBKEY 112695A0E562B32A
 supported_app_version: ["3.11", "4.0"]
 expected_metrics:
   - type: workload.googleapis.com/cassandra.client.request.count

integration_test/third_party_apps_data/applications/couchbase/metadata.yaml

@@ -200,9 +200,9 @@ supported_operating_systems: linux
 platforms_to_skip:
   # couchbase is not supported on various distros.
   # https://docs.couchbase.com/server/current/install/install-platforms.html
-  - rocky-linux-9
-  - debian-11-arm64
-  - debian-12
-  - ubuntu-2310-amd64
-  - ubuntu-2310-arm64
+  - rocky-linux-cloud:rocky-linux-9
+  - debian-cloud:debian-11-arm64
+  - debian-cloud:debian-12
+  - ubuntu-os-cloud:ubuntu-2310-amd64
+  - ubuntu-os-cloud:ubuntu-2310-arm64
 public_url: https://cloud.google.com/stackdriver/docs/solutions/agents/ops-agent/third-party/couchbase

integration_test/third_party_apps_data/applications/couchdb/metadata.yaml

@@ -29,15 +29,15 @@ minimum_supported_agent_version:
 supported_operating_systems: linux
 platforms_to_skip:
   # couchdb is not supported on various distros.
-  - sles-12
-  - sles-15
-  - sles-15-arm64
-  - rocky-linux-9
-  - rocky-linux-9-arm64
-  - ubuntu-2310-amd64
-  - ubuntu-2310-arm64
-  - debian-12
-  - debian-12-arm64
+  - suse-cloud:sles-12
+  - suse-cloud:sles-15
+  - suse-cloud:sles-15-arm64
+  - rocky-linux-cloud:rocky-linux-9
+  - rocky-linux-cloud:rocky-linux-9-arm64
+  - ubuntu-os-cloud:ubuntu-2310-amd64
+  - ubuntu-os-cloud:ubuntu-2310-arm64
+  - debian-cloud:debian-12
+  - debian-cloud:debian-12-arm64
 supported_app_version: ["2.3+", "3.1+"]
 expected_metrics:
   - type: workload.googleapis.com/couchdb.average_request_time