[v3] Schema Version upgrading for v1 and v2. (#5796)

1. Add upgrade func to v3alpha1
2. Enable schema parsing for v2 schemas (v3alpha*)
3. Compatibility check among v1 and v2 schemas
4. Update `skaffold fix` to avoid upgrading v1 schemas to v2 and vice versa.
5. Update the schemas usage in `hack`, `cmd/fix` `cmd/list` to hide v2 schemas from users.

cmd/skaffold/app/cmd/fix.go

@@ -67,10 +67,16 @@ func fix(out io.Writer, configFile string, toVersion string, overwrite bool) err
 		return nil
 	}
 
-	versionedCfgs, err := schema.ParseConfigAndUpgrade(configFile, toVersion)
+	versionedCfgs, err := schema.ParseConfig(configFile)
 	if err != nil {
 		return err
 	}
+	if ok, err := schema.IsCompatibleWith(versionedCfgs, toVersion); !ok {
+		return err
+	}
+	if versionedCfgs, err = schema.UpgradeTo(versionedCfgs, toVersion); err != nil {
+		return err
+	}
 
 	// TODO(dgageot): We should be able run validations on any schema version
 	// but that's not the case. They can only run on the latest version for now.

cmd/skaffold/app/cmd/schema/list.go

@@ -27,6 +27,7 @@ import (
 
 var OutputType string
 
+// TODO(yuwenma): We currently hide the v3alpha* schemas from users. We should display it once the render v2 is available.
 // List prints to `out` all supported schema versions.
 func List(_ context.Context, out io.Writer) error {
 	return list(out, OutputType)
@@ -64,7 +65,7 @@ func printPlain(out io.Writer) error {
 func versions() []string {
 	var versions []string
 
-	for _, version := range schema.SchemaVersions {
+	for _, version := range schema.SchemaVersionsV1 {
 		versions = append(versions, version.APIVersion)
 	}
 

hack/schemas/main.go

@@ -93,17 +93,18 @@ type sameErr struct {
 	err  error
 }
 
+// TODO(yuwenma): Generate v2 schemas.
 func generateSchemas(root string, dryRun bool) (bool, error) {
 	var results [](chan sameErr)
-	for range schema.SchemaVersions {
+	for range schema.AllVersions {
 		results = append(results, make(chan sameErr, 1))
 	}
 
 	var wg sync.WaitGroup
-	for i, version := range schema.SchemaVersions {
+	for i, version := range schema.SchemaVersionsV1 {
 		wg.Add(1)
 		go func(i int, version schema.Version) {
-			same, err := generateSchema(root, dryRun, version)
+			same, err := generateV1Schema(root, dryRun, version)
 			results[i] <- sameErr{
 				same: same,
 				err:  err,
@@ -114,7 +115,7 @@ func generateSchemas(root string, dryRun bool) (bool, error) {
 	wg.Wait()
 
 	same := true
-	for i := range schema.SchemaVersions {
+	for i := range schema.SchemaVersionsV1 {
 		result := <-results[i]
 		if result.err != nil {
 			return false, result.err
@@ -126,12 +127,12 @@ func generateSchemas(root string, dryRun bool) (bool, error) {
 	return same, nil
 }
 
-func generateSchema(root string, dryRun bool, version schema.Version) (bool, error) {
+func generateV1Schema(root string, dryRun bool, version schema.Version) (bool, error) {
 	apiVersion := strings.TrimPrefix(version.APIVersion, "skaffold/")
 
 	folder := apiVersion
 	strict := false
-	if version.APIVersion == schema.SchemaVersions[len(schema.SchemaVersions)-1].APIVersion {
+	if version.APIVersion == schema.SchemaVersionsV1[len(schema.SchemaVersionsV1)-1].APIVersion {
 		folder = "latest/v1"
 		strict = true
 	}

hack/versions/cmd/latest/version.go

@@ -24,6 +24,7 @@ import (
 	"github.com/GoogleContainerTools/skaffold/hack/versions/pkg/schema"
 )
 
+// TODO(yuwenma): Print the latest versions for both v1 and v2.
 // Print the latest version released.
 func main() {
 	logrus.SetLevel(logrus.ErrorLevel)

hack/versions/cmd/latest_released/version.go

@@ -26,6 +26,7 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema"
 )
 
+// TODO(yuwenma): Print the latest released versions for both v1 and v2.
 // Print the latest version released.
 func main() {
 	logrus.SetLevel(logrus.ErrorLevel)
@@ -35,7 +36,7 @@ func main() {
 	if latestIsReleased {
 		fmt.Println(current)
 	} else {
-		prev := strings.TrimPrefix(schema.SchemaVersions[len(schema.SchemaVersions)-2].APIVersion, "skaffold/")
+		prev := strings.TrimPrefix(schema.SchemaVersionsV1[len(schema.SchemaVersionsV1)-2].APIVersion, "skaffold/")
 		fmt.Println(prev)
 	}
 }

hack/versions/cmd/new/version.go

@@ -34,11 +34,12 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/walk"
 )
 
+// TODO(yuwenma): Upgrade the version to include v3alpha* once it's available.
 // Before: prev -> current (latest)
 // After:  prev -> current -> new (latest)
 func main() {
 	logrus.SetLevel(logrus.DebugLevel)
-	prev := strings.TrimPrefix(schema.SchemaVersions[len(schema.SchemaVersions)-2].APIVersion, "skaffold/")
+	prev := strings.TrimPrefix(schema.SchemaVersionsV1[len(schema.SchemaVersionsV1)-2].APIVersion, "skaffold/")
 	logrus.Infof("Previous Skaffold version: %s", prev)
 
 	current, latestIsReleased := hackschema.GetLatestVersion()

pkg/skaffold/parser/config.go

@@ -97,7 +97,7 @@ func GetConfigSet(opts config.SkaffoldOptions) (SkaffoldConfigSet, error) {
 
 // getConfigs recursively parses all configs and their dependencies in the specified `skaffold.yaml`
 func getConfigs(cfgOpts configOpts, opts config.SkaffoldOptions, r *record) (SkaffoldConfigSet, error) {
-	parsed, err := schema.ParseConfigAndUpgrade(cfgOpts.file, latest_v1.Version)
+	parsed, err := schema.ParseConfigAndUpgrade(cfgOpts.file)
 	if err != nil {
 		if errors.Is(err, os.ErrNotExist) {
 			return nil, sErrors.MainConfigFileNotFoundErr(cfgOpts.file, err)

pkg/skaffold/runner/v1/generate_pipeline.go

@@ -74,7 +74,7 @@ func setupConfigFiles(configPaths []string) ([]*pipeline.ConfigFile, error) {
 	// Read all given config files to read contents into SkaffoldConfig
 	var configFiles []*pipeline.ConfigFile
 	for _, path := range configPaths {
-		parsedCfgs, err := schema.ParseConfigAndUpgrade(path, latest_v1.Version)
+		parsedCfgs, err := schema.ParseConfigAndUpgrade(path)
 		if err != nil {
 			return nil, fmt.Errorf("parsing config %q: %w", path, err)
 		}

pkg/skaffold/schema/latest/v2/config.go

@@ -21,7 +21,7 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/util"
 )
 
-const Version string = "skaffold/v3alpha1"
+const Version string = "skaffold/v3alpha2"
 
 // NewSkaffoldConfig creates a SkaffoldConfig
 func NewSkaffoldConfig() util.VersionedConfig {

pkg/skaffold/schema/samples_test.go

@@ -77,7 +77,7 @@ func TestParseSamples(t *testing.T) {
 
 func checkSkaffoldConfig(t *testutil.T, yaml []byte) {
 	configFile := t.TempFile("skaffold.yaml", yaml)
-	parsed, err := ParseConfigAndUpgrade(configFile, latest_v1.Version)
+	parsed, err := ParseConfigAndUpgrade(configFile)
 	t.CheckNoError(err)
 	var cfgs []*latest_v1.SkaffoldConfig
 	for _, p := range parsed {

pkg/skaffold/schema/v3alpha1/upgrade.go

@@ -17,11 +17,22 @@ limitations under the License.
 package v3alpha1
 
 import (
+	next "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v2"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/util"
+	pkgutil "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 )
 
 // Upgrade upgrades a configuration to the next version.
 // Config changes from v3alpha1 to v3alpha2
 func (c *SkaffoldConfig) Upgrade() (util.VersionedConfig, error) {
-	return c, nil
+	var newConfig next.SkaffoldConfig
+	pkgutil.CloneThroughJSON(c, &newConfig)
+	newConfig.APIVersion = next.Version
+
+	err := util.UpgradePipelines(c, &newConfig, upgradeOnePipeline)
+	return &newConfig, err
+}
+
+func upgradeOnePipeline(oldPipeline, newPipeline interface{}) error {
+	return nil
 }

pkg/skaffold/schema/versions.go

@@ -21,14 +21,16 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"regexp"
 	"strings"
 
 	"github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v3"
 
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/apiversion"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/kubernetes"
-	latest_v1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
+	latestV1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v1"
+	latestV2 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest/v2"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/util"
 	v1 "github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v1alpha1"
@@ -72,14 +74,23 @@ import (
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v2beta7"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v2beta8"
 	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v2beta9"
+	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/v3alpha1"
 	misc "github.com/GoogleContainerTools/skaffold/pkg/skaffold/util"
 )
 
+var (
+	AllVersions = append(SchemaVersionsV1, SchemaVersionsV2...)
+	V1Pattern   = regexp.MustCompile(`skaffold/v[12]((alpha|beta)\d+)?`)
+	V2Pattern   = regexp.MustCompile(`skaffold/v3((alpha|beta)\d+)?`)
+)
+
 type APIVersion struct {
 	Version string `yaml:"apiVersion"`
 }
 
-var SchemaVersions = Versions{
+// SchemaVersionsV1 refers to all the supported API Schemas for skaffold v1 executables.e.g. skaffold 1.13. The API
+// schema versions are in the range of v1alpha*, v1beta*, v2alpha* and v2beta*.
+var SchemaVersionsV1 = Versions{
 	{v1alpha1.Version, v1alpha1.NewSkaffoldConfig},
 	{v1alpha2.Version, v1alpha2.NewSkaffoldConfig},
 	{v1alpha3.Version, v1alpha3.NewSkaffoldConfig},
@@ -122,7 +133,14 @@ var SchemaVersions = Versions{
 	{v2beta13.Version, v2beta13.NewSkaffoldConfig},
 	{v2beta14.Version, v2beta14.NewSkaffoldConfig},
 	{v2beta15.Version, v2beta15.NewSkaffoldConfig},
-	{latest_v1.Version, latest_v1.NewSkaffoldConfig},
+	{latestV1.Version, latestV1.NewSkaffoldConfig},
+}
+
+// SchemaVersionsV2 refers to all the supported API Schemas for skaffold v2 executables. The API schema versions are
+// in the range of v3alpha*.
+var SchemaVersionsV2 = Versions{
+	{v3alpha1.Version, v1alpha1.NewSkaffoldConfig},
+	{latestV2.Version, latestV2.NewSkaffoldConfig},
 }
 
 type Version struct {
@@ -173,51 +191,16 @@ func ParseConfig(filename string) ([]util.VersionedConfig, error) {
 }
 
 // ParseConfigAndUpgrade reads a configuration file and upgrades it to a given version.
-func ParseConfigAndUpgrade(filename, toVersion string) ([]util.VersionedConfig, error) {
+func ParseConfigAndUpgrade(filename string) ([]util.VersionedConfig, error) {
 	configs, err := ParseConfig(filename)
 	if err != nil {
 		return nil, err
 	}
-
-	// Check that the target version exists
-	if _, present := SchemaVersions.Find(toVersion); !present {
-		return nil, fmt.Errorf("unknown api version: %q", toVersion)
-	}
-	upgradeNeeded := false
-	for _, cfg := range configs {
-		// Check that the config's version is not newer than the target version
-		currentVersion, err := apiversion.Parse(cfg.GetVersion())
-		if err != nil {
-			return nil, err
-		}
-		targetVersion, err := apiversion.Parse(toVersion)
-		if err != nil {
-			return nil, err
-		}
-
-		if currentVersion.NE(targetVersion) {
-			upgradeNeeded = true
-		}
-		if currentVersion.GT(targetVersion) {
-			return nil, fmt.Errorf("config version %q is more recent than target version %q: upgrade Skaffold", cfg.GetVersion(), toVersion)
-		}
-	}
-	if !upgradeNeeded {
-		return configs, nil
-	}
-	logrus.Debugf("config version out of date: upgrading to latest %q", toVersion)
-
-	var upgraded []util.VersionedConfig
-	for _, cfg := range configs {
-		for cfg.GetVersion() != toVersion {
-			cfg, err = cfg.Upgrade()
-			if err != nil {
-				return nil, fmt.Errorf("transforming skaffold config: %w", err)
-			}
-		}
-		upgraded = append(upgraded, cfg)
+	toVersion, err := getLatestFromCompatibilityCheck(configs)
+	if err != nil {
+		return nil, err
 	}
-	return upgraded, nil
+	return UpgradeTo(configs, toVersion)
 }
 
 // configFactoryFromAPIVersion checks that all configs in the input stream have the same API version, and returns a function to create a config with that API version.
@@ -240,7 +223,7 @@ func configFactoryFromAPIVersion(buf []byte) ([]func() util.VersionedConfig, err
 		if err != nil {
 			return nil, fmt.Errorf("parsing api version: %w", err)
 		}
-		factory, present := SchemaVersions.Find(v.Version)
+		factory, present := AllVersions.Find(v.Version)
 		if !present {
 			return nil, fmt.Errorf("unknown api version: %q", v.Version)
 		}
@@ -301,3 +284,94 @@ func parseConfig(buf []byte, factories []func() util.VersionedConfig) ([]util.Ve
 	}
 	return cfgs, nil
 }
+
+// getLatestFromCompatibilityCheck makes sure the schema versions in SchemaVersionsV1 and SchemaVersionsV2 are not used
+// together and returns the latest version where this VersionedConfig slice belongs to.
+func getLatestFromCompatibilityCheck(cfgs []util.VersionedConfig) (string, error) {
+	var v1Track, v2Track []string
+	for _, cfg := range cfgs {
+		curVersion := cfg.GetVersion()
+		if matched := V1Pattern.MatchString(curVersion); matched {
+			v1Track = append(v1Track, curVersion)
+		} else if matched := V2Pattern.MatchString(curVersion); matched {
+			v2Track = append(v2Track, curVersion)
+		} else {
+			return "", fmt.Errorf("unknown apiVersion %v", curVersion)
+		}
+	}
+
+	if len(v1Track) > 0 && len(v2Track) > 0 {
+		return "", fmt.Errorf("detected incompatible versions:%v are incompatible with %v", v1Track, v2Track)
+	}
+	if len(v1Track) > 0 {
+		return latestV1.Version, nil
+	}
+	if len(v2Track) > 0 {
+		return latestV2.Version, nil
+	}
+	return "", fmt.Errorf("unable to find a valid API Schema version")
+}
+
+// IsCompatibleWith checks if the cfgs versions can be upgraded to toVersion.
+func IsCompatibleWith(cfgs []util.VersionedConfig, toVersion string) (bool, error) {
+	var pattern *regexp.Regexp
+	if matched := V1Pattern.MatchString(toVersion); matched {
+		pattern = V1Pattern
+	} else if matched := V2Pattern.MatchString(toVersion); matched {
+		pattern = V2Pattern
+	} else {
+		return false, fmt.Errorf("target version %v is invalid", toVersion)
+	}
+	var badVersions []string
+	for _, cfg := range cfgs {
+		curVersion := cfg.GetVersion()
+		if matched := pattern.MatchString(curVersion); !matched {
+			badVersions = append(badVersions, curVersion)
+		}
+	}
+	if len(badVersions) > 0 {
+		return false, fmt.Errorf(
+			"the following versions are incompatible with target version %v. upgrade aborted",
+			badVersions)
+	}
+	return true, nil
+}
+
+// UpgradeTo upgrades the given configs to toVersion.
+func UpgradeTo(configs []util.VersionedConfig, toVersion string) ([]util.VersionedConfig, error) {
+	upgradeNeeded := false
+	for _, cfg := range configs {
+		// Check that the config's version is not newer than the target version
+		currentVersion, err := apiversion.Parse(cfg.GetVersion())
+		if err != nil {
+			return nil, err
+		}
+		targetVersion, err := apiversion.Parse(toVersion)
+		if err != nil {
+			return nil, err
+		}
+
+		if currentVersion.NE(targetVersion) {
+			upgradeNeeded = true
+		}
+		if currentVersion.GT(targetVersion) {
+			return nil, fmt.Errorf("config version %q is more recent than target version %q: upgrade Skaffold", cfg.GetVersion(), toVersion)
+		}
+	}
+	if !upgradeNeeded {
+		return configs, nil
+	}
+	logrus.Debugf("config version out of date: upgrading to latest %q", toVersion)
+	var err error
+	var upgraded []util.VersionedConfig
+	for _, cfg := range configs {
+		for cfg.GetVersion() != toVersion {
+			cfg, err = cfg.Upgrade()
+			if err != nil {
+				return nil, fmt.Errorf("transforming skaffold config: %w", err)
+			}
+		}
+		upgraded = append(upgraded, cfg)
+	}
+	return upgraded, nil
+}