Merge pull request #146 from ulucinar/local-scheme

Use a local runtime.Scheme with the migration.Registry

go.mod

@@ -29,6 +29,7 @@ require (
 	k8s.io/client-go v0.25.0
 	k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed
 	sigs.k8s.io/controller-runtime v0.12.1
+	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2
 	sigs.k8s.io/yaml v1.3.0
 )
 
@@ -102,6 +103,5 @@ require (
 	k8s.io/component-base v0.25.0 // indirect
 	k8s.io/klog/v2 v2.70.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20220803162953-67bda5d908f1 // indirect
-	sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )

pkg/migration/converter.go

@@ -16,11 +16,13 @@ package migration
 
 import (
 	"github.com/crossplane/crossplane-runtime/pkg/fieldpath"
+	xpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1"
 	"github.com/pkg/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/json"
+	k8sjson "sigs.k8s.io/json"
 )
 
 const (
@@ -110,3 +112,14 @@ func FromGroupVersionKind(gvk schema.GroupVersionKind) GroupVersionKind {
 		Kind:    gvk.Kind,
 	}
 }
+
+// workaround for:
+// https://github.com/kubernetes-sigs/structured-merge-diff/issues/230
+func convertToComposition(u map[string]interface{}) (*xpv1.Composition, error) {
+	buff, err := json.Marshal(u)
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to marshal map to JSON")
+	}
+	c := &xpv1.Composition{}
+	return c, errors.Wrap(k8sjson.UnmarshalCaseSensitivePreserveInts(buff, c), "failed to unmarshal into a v1.Composition")
+}

pkg/migration/filesystem.go

@@ -121,6 +121,9 @@ func (ft *FileSystemTarget) Put(o UnstructuredWithMetadata) error {
 	if err != nil {
 		return errors.Wrap(err, "cannot marshal object")
 	}
+	if err := os.MkdirAll(filepath.Dir(o.Metadata.Path), 0o750); err != nil {
+		return errors.Wrapf(err, "cannot mkdirall: %s", filepath.Dir(o.Metadata.Path))
+	}
 	if o.Metadata.Parents != "" {
 		f, err := ft.afero.OpenFile(o.Metadata.Path, os.O_APPEND|os.O_WRONLY|os.O_CREATE, 0600)
 		if err != nil {

pkg/migration/kubernetes.go

@@ -2,9 +2,9 @@ package migration
 
 import (
 	"context"
-	"strings"
 
 	"github.com/pkg/errors"
+	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
@@ -27,32 +27,46 @@ type KubernetesSource struct {
 // Group:   "ec2.aws.upbound.io",
 // Version: "v1beta1",
 // Kind:    "VPC",
-func NewKubernetesSource(dynamicClient dynamic.Interface, gvks []schema.GroupVersionKind) (*KubernetesSource, error) {
+func NewKubernetesSource(r *Registry, dynamicClient dynamic.Interface) (*KubernetesSource, error) {
 	ks := &KubernetesSource{
 		dynamicClient: dynamicClient,
 	}
+	if err := ks.getResources(r.claimTypes, CategoryClaim); err != nil {
+		return nil, errors.Wrap(err, "cannot get claims")
+	}
+	if err := ks.getResources(r.compositeTypes, CategoryComposite); err != nil {
+		return nil, errors.Wrap(err, "cannot get composites")
+	}
+	if err := ks.getResources(r.GetCompositionGVKs(), CategoryComposition); err != nil {
+		return nil, errors.Wrap(err, "cannot get compositions")
+	}
+	if err := ks.getResources(r.GetManagedResourceGVKs(), CategoryManaged); err != nil {
+		return nil, errors.Wrap(err, "cannot get managed resources")
+	}
+	return ks, nil
+}
+
+func (ks *KubernetesSource) getResources(gvks []schema.GroupVersionKind, category Category) error {
 	for _, gvk := range gvks {
-		ri := dynamicClient.Resource(
-			schema.GroupVersionResource{
-				Group:   gvk.Group,
-				Version: gvk.Version,
-				// we need to add plural appendix to end of kind name
-				Resource: strings.ToLower(gvk.Kind) + "s",
-			})
+		// TODO: we are not using discovery as of now (to be reconsidered).
+		// This will not in all cases.
+		pluralGVR, _ := meta.UnsafeGuessKindToResource(gvk)
+		ri := ks.dynamicClient.Resource(pluralGVR)
 		unstructuredList, err := ri.List(context.TODO(), metav1.ListOptions{})
 		if err != nil {
-			return nil, errors.Wrap(err, "cannot list resources")
+			return errors.Wrap(err, "cannot list resources")
 		}
 		for _, u := range unstructuredList.Items {
 			ks.items = append(ks.items, UnstructuredWithMetadata{
 				Object: u,
 				Metadata: Metadata{
-					Path: string(u.GetUID()),
+					Path:     string(u.GetUID()),
+					Category: category,
 				},
 			})
 		}
 	}
-	return ks, nil
+	return nil
 }
 
 // HasNext checks the next item

pkg/migration/kubernetes_test.go

@@ -40,6 +40,9 @@ func TestNewKubernetesSource(t *testing.T) {
 							Object: unstructured.Unstructured{
 								Object: unstructuredAwsVpc,
 							},
+							Metadata: Metadata{
+								Category: CategoryManaged,
+							},
 						},
 					},
 				},
@@ -48,10 +51,14 @@ func TestNewKubernetesSource(t *testing.T) {
 	}
 	for name, tc := range cases {
 		t.Run(name, func(t *testing.T) {
-			dynamicClient := fake.NewSimpleDynamicClient(runtime.NewScheme(),
+			s := runtime.NewScheme()
+			r := NewRegistry(s)
+			// register a dummy converter so that MRs will be observed
+			r.converters = map[schema.GroupVersionKind]Converter{tc.args.gvks[0]: nil}
+			dynamicClient := fake.NewSimpleDynamicClient(s,
 				&unstructured.Unstructured{Object: unstructuredAwsVpc},
 				&unstructured.Unstructured{Object: unstructuredResourceGroup})
-			ks, err := NewKubernetesSource(dynamicClient, tc.args.gvks)
+			ks, err := NewKubernetesSource(r, dynamicClient)
 			if diff := cmp.Diff(tc.want.err, err); diff != "" {
 				t.Errorf("\nNext(...): -want, +got:\n%s", diff)
 			}

pkg/migration/plan_generator.go

@@ -18,19 +18,18 @@ import (
 	"fmt"
 	"strings"
 
-	v1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
-	"github.com/crossplane/crossplane-runtime/pkg/fieldpath"
-	"github.com/crossplane/crossplane-runtime/pkg/resource/unstructured/claim"
-	"github.com/crossplane/crossplane-runtime/pkg/resource/unstructured/composite"
+	"github.com/pkg/errors"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/client-go/kubernetes/scheme"
 
+	v1 "github.com/crossplane/crossplane-runtime/apis/common/v1"
+	"github.com/crossplane/crossplane-runtime/pkg/fieldpath"
 	"github.com/crossplane/crossplane-runtime/pkg/meta"
 	"github.com/crossplane/crossplane-runtime/pkg/resource"
+	"github.com/crossplane/crossplane-runtime/pkg/resource/unstructured/claim"
+	"github.com/crossplane/crossplane-runtime/pkg/resource/unstructured/composite"
 	xpv1 "github.com/crossplane/crossplane/apis/apiextensions/v1"
-	"github.com/pkg/errors"
 )
 
 const (
@@ -71,22 +70,26 @@ const (
 	stepStartComposites
 )
 
+const (
+	versionV010 = "0.1.0"
+)
+
 // PlanGenerator generates a migration.Plan reading the manifests available
 // from `source`, converting managed resources and compositions using the
 // available `migration.Converter`s registered in the `registry` and
 // writing the output manifests to the specified `target`.
 type PlanGenerator struct {
 	source   Source
 	target   Target
-	registry Registry
+	registry *Registry
 	// Plan is the migration.Plan whose steps are expected
 	// to complete a migration when they're executed in order.
 	Plan Plan
 }
 
 // NewPlanGenerator constructs a new PlanGenerator using the specified
 // Source and Target and the default converter Registry.
-func NewPlanGenerator(source Source, target Target) PlanGenerator {
+func NewPlanGenerator(registry *Registry, source Source, target Target) PlanGenerator {
 	return PlanGenerator{
 		source:   source,
 		target:   target,
@@ -135,15 +138,15 @@ func (pg *PlanGenerator) convert() error { //nolint: gocyclo
 				}
 			}
 		default:
-			if o.Metadata.IsComposite {
+			if o.Metadata.Category == CategoryComposite {
 				if err := pg.stepPauseComposite(&o); err != nil {
 					return errors.Wrap(err, errCompositePause)
 				}
 				composites = append(composites, o)
 				continue
 			}
 
-			if o.Metadata.IsClaim {
+			if o.Metadata.Category == CategoryClaim {
 				claims = append(claims, o)
 				continue
 			}
@@ -167,7 +170,7 @@ func (pg *PlanGenerator) convert() error { //nolint: gocyclo
 						return errors.Wrap(err, errResourceMigrate)
 					}
 				}
-			} else if _, ok, _ := toManagedResource(o.Object); ok {
+			} else if _, ok, _ := toManagedResource(pg.registry.scheme, o.Object); ok {
 				if err := pg.stepStartManagedResource(&o); err != nil {
 					return errors.Wrap(err, errResourceMigrate)
 				}
@@ -191,12 +194,12 @@ func (pg *PlanGenerator) convert() error { //nolint: gocyclo
 
 func (pg *PlanGenerator) convertResource(o UnstructuredWithMetadata) ([]UnstructuredWithMetadata, bool, error) {
 	gvk := o.Object.GroupVersionKind()
-	conv := pg.registry[gvk]
+	conv := pg.registry.converters[gvk]
 	if conv == nil {
 		return []UnstructuredWithMetadata{o}, false, nil
 	}
 	// we have already ensured that the GVK belongs to a managed resource type
-	mg, _, err := toManagedResource(o.Object)
+	mg, _, err := toManagedResource(pg.registry.scheme, o.Object)
 	if err != nil {
 		return nil, false, errors.Wrap(err, errResourceMigrate)
 	}
@@ -214,9 +217,12 @@ func (pg *PlanGenerator) convertResource(o UnstructuredWithMetadata) ([]Unstruct
 	return converted, true, nil
 }
 
-func toManagedResource(u unstructured.Unstructured) (resource.Managed, bool, error) {
+func toManagedResource(c runtime.ObjectCreater, u unstructured.Unstructured) (resource.Managed, bool, error) {
 	gvk := u.GroupVersionKind()
-	obj, err := scheme.Scheme.New(gvk)
+	if gvk == xpv1.CompositionGroupVersionKind {
+		return nil, false, nil
+	}
+	obj, err := c.New(gvk)
 	if err != nil {
 		return nil, false, errors.Wrapf(err, errFmtNewObject, gvk)
 	}
@@ -228,8 +234,8 @@ func toManagedResource(u unstructured.Unstructured) (resource.Managed, bool, err
 }
 
 func (pg *PlanGenerator) convertComposition(o UnstructuredWithMetadata) (*UnstructuredWithMetadata, bool, error) { // nolint:gocyclo
-	c := xpv1.Composition{}
-	if err := runtime.DefaultUnstructuredConverter.FromUnstructured(o.Object.Object, &c); err != nil {
+	c, err := convertToComposition(o.Object.Object)
+	if err != nil {
 		return nil, false, errors.Wrap(err, errUnstructuredConvert)
 	}
 	var targetResources []*xpv1.ComposedTemplate
@@ -260,7 +266,7 @@ func (pg *PlanGenerator) convertComposition(o UnstructuredWithMetadata) (*Unstru
 			}
 			cmps = append(cmps, c)
 		}
-		conv := pg.registry[gvk]
+		conv := pg.registry.converters[gvk]
 		if conv != nil {
 			if err := conv.ComposedTemplates(cmp, cmps...); err != nil {
 				return nil, false, errors.Wrap(err, errComposedTemplateMigrate)
@@ -328,10 +334,11 @@ func (pg *PlanGenerator) buildPlan() {
 	pg.Plan.Spec.Steps[stepStartComposites].Name = "start-composites"
 	pg.Plan.Spec.Steps[stepStartComposites].Type = StepTypeApply
 	pg.Plan.Spec.Steps[stepStartComposites].Apply = &ApplyStep{}
+	pg.Plan.Version = versionV010
 }
 
 func (pg *PlanGenerator) addStepsForManagedResource(u *UnstructuredWithMetadata) error {
-	if _, ok, err := toManagedResource(u.Object); err != nil || !ok {
+	if _, ok, err := toManagedResource(pg.registry.scheme, u.Object); err != nil || !ok {
 		// not a managed resource or unable to determine
 		// whether it's a managed resource
 		return nil // nolint:nilerr

pkg/migration/plan_generator_test.go

@@ -28,9 +28,9 @@ import (
 	v1 "github.com/crossplane/crossplane/apis/apiextensions/v1"
 	"github.com/google/go-cmp/cmp"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/util/yaml"
-	"k8s.io/client-go/kubernetes/scheme"
 	k8syaml "sigs.k8s.io/yaml"
 
 	"github.com/upbound/upjet/pkg/migration/fake"
@@ -40,7 +40,7 @@ func TestGeneratePlan(t *testing.T) {
 	type fields struct {
 		source   Source
 		target   *testTarget
-		registry Registry
+		registry *Registry
 	}
 	type want struct {
 		err               error
@@ -56,17 +56,17 @@ func TestGeneratePlan(t *testing.T) {
 			fields: fields{
 				source: newTestSource(map[string]Metadata{
 					"testdata/plan/sourcevpc.yaml":   {},
-					"testdata/plan/claim.yaml":       {IsClaim: true},
+					"testdata/plan/claim.yaml":       {Category: CategoryClaim},
 					"testdata/plan/composition.yaml": {},
 					"testdata/plan/xrd.yaml":         {},
-					"testdata/plan/xr.yaml":          {IsComposite: true}}),
+					"testdata/plan/xr.yaml":          {Category: CategoryComposite}}),
 				target: newTestTarget(),
 				registry: getRegistryWithConverters(map[schema.GroupVersionKind]Converter{
 					fake.MigrationSourceGVK: &testConverter{},
 				}),
 			},
 			want: want{
-				migrationPlanPath: "testdata/plan/migration_plan.yaml",
+				migrationPlanPath: "testdata/plan/generated/migration_plan.yaml",
 				migratedResourceNames: []string{
 					"pause-managed/sample-vpc.vpcs.fakesourceapi.yaml",
 					"edit-claims/my-resource.myresources.test.com.yaml",
@@ -83,7 +83,7 @@ func TestGeneratePlan(t *testing.T) {
 	}
 	for name, tt := range tests {
 		t.Run(name, func(t *testing.T) {
-			pg := NewPlanGenerator(tt.fields.source, tt.fields.target)
+			pg := NewPlanGenerator(tt.fields.registry, tt.fields.source, tt.fields.target)
 			err := pg.GeneratePlan()
 			// compare error state
 			if diff := cmp.Diff(tt.want.err, err, test.EquateErrors()); diff != "" {
@@ -244,12 +244,14 @@ func (f *testConverter) ComposedTemplates(cmp v1.ComposedTemplate, convertedBase
 	return nil
 }
 
-func getRegistryWithConverters(converters map[schema.GroupVersionKind]Converter) Registry {
-	scheme.Scheme.AddKnownTypeWithName(fake.MigrationSourceGVK, &fake.MigrationSourceObject{})
+func getRegistryWithConverters(converters map[schema.GroupVersionKind]Converter) *Registry {
+	scheme := runtime.NewScheme()
+	scheme.AddKnownTypeWithName(fake.MigrationSourceGVK, &fake.MigrationSourceObject{})
+	r := NewRegistry(scheme)
 	for gvk, c := range converters {
-		RegisterConverter(gvk, c)
+		r.RegisterConverter(gvk, c)
 	}
-	return registry
+	return r
 }
 
 func loadPlan(planPath string) (*Plan, error) {