Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

✨ Support IPAM and runtime extension providers #7288

Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -612,8 +612,8 @@ docker-capd-build-all: $(addprefix docker-capd-build-,$(ALL_ARCH)) ## Build capd
.PHONY: docker-build-test-extension
docker-build-test-extension: ## Build the docker image for core controller manager
DOCKER_BUILDKIT=1 docker build --build-arg builder_image=$(GO_CONTAINER_IMAGE) --build-arg goproxy=$(GOPROXY) --build-arg ARCH=$(ARCH) --build-arg ldflags="$(LDFLAGS)" . -t $(TEST_EXTENSION_IMG)-$(ARCH):$(TAG) --file ./test/extension/Dockerfile
$(MAKE) set-manifest-image MANIFEST_IMG=$(TEST_EXTENSION_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./test/extension/config/default/extension_image_patch.yaml"
$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./test/extension/config/default/extension_pull_policy.yaml"
$(MAKE) set-manifest-image MANIFEST_IMG=$(TEST_EXTENSION_IMG)-$(ARCH) MANIFEST_TAG=$(TAG) TARGET_RESOURCE="./test/extension/config/default/manager_image_patch.yaml"
$(MAKE) set-manifest-pull-policy TARGET_RESOURCE="./test/extension/config/default/manager_pull_policy.yaml"

.PHONY: e2e-framework
e2e-framework: ## Builds the CAPI e2e framework
Expand Down Expand Up @@ -710,6 +710,7 @@ tilt-up: kind-cluster ## Start tilt and build kind cluster if needed.
docker-build-e2e: ## Rebuild all Cluster API provider images to be used in the e2e tests
$(MAKE) docker-build REGISTRY=gcr.io/k8s-staging-cluster-api PULL_POLICY=IfNotPresent
$(MAKE) docker-capd-build REGISTRY=gcr.io/k8s-staging-cluster-api PULL_POLICY=IfNotPresent
$(MAKE) docker-build-test-extension REGISTRY=gcr.io/k8s-staging-cluster-api PULL_POLICY=IfNotPresent

## --------------------------------------
## Release
Expand Down
115 changes: 19 additions & 96 deletions Tiltfile
Original file line number Diff line number Diff line change
Expand Up @@ -104,30 +104,30 @@ providers = {
"internal",
"third_party",
],
"label": "CAPD",
# Add kubectl to the docker image, CAPD manager requires it.
"additional_docker_helper_commands": "RUN curl -LO https://dl.k8s.io/release/{KUBE}/bin/linux/{ARCH}/kubectl && chmod +x ./kubectl && mv ./kubectl /usr/bin/kubectl".format(
ARCH = os_arch,
KUBE = kubernetes_version,
),
"additional_docker_build_commands": """
COPY --from=tilt-helper /usr/bin/kubectl /usr/bin/kubectl
""",
"label": "CAPD",
},
}

# Create a data structure to hold information about addons.
addons = {
"test-extension": {
"context": "./test/extension",
"context": "test/extension",
"image": "gcr.io/k8s-staging-cluster-api/test-extension",
"container_name": "extension",
"live_reload_deps": ["main.go", "handlers"],
"live_reload_deps": [
"main.go",
"handlers",
],
"label": "test-extension",
"resource_deps": ["capi_controller"],
# Add the ExtensionConfig for this Runtime extension; given that the ExtensionConfig can be installed only when capi_controller
# are up and running, it is required to set a resource_deps to ensure proper install order.
"additional_resources": [
"config/tilt/extensionconfig.yaml",
"config/tilt/hookresponses-configmap.yaml",
],
"resource_deps": ["capi_controller"],
},
}

Expand Down Expand Up @@ -165,27 +165,6 @@ def load_provider_tiltfiles():
provider_config["go_main"] = "main.go"
providers[provider_name] = provider_config

# load_addon_tiltfiles looks for tilt-addon.[yaml|json] files in the repositories listed in "addon_repos" in tilt settings and loads their config.
def load_addon_tiltfiles():
addon_repos = settings.get("addon_repos", [])
for repo in addon_repos:
file = repo + "/tilt-addon.yaml" if os.path.exists(repo + "/tilt-addon.yaml") else repo + "/tilt-addon.json"
if not os.path.exists(file):
fail("Failed to load provider. No tilt-addon.{yaml|json} file found in " + repo)
addon_details = read_yaml(file, default = {})
if type(addon_details) != type([]):
addon_details = [addon_details]
for item in addon_details:
addon_name = item["name"]
addon_config = item["config"]
if "context" in addon_config:
addon_config["context"] = repo + "/" + addon_config["context"]
else:
addon_config["context"] = repo
if "go_main" not in addon_config:
addon_config["go_main"] = "main.go"
addons[addon_name] = addon_config

tilt_helper_dockerfile_header = """
# Tilt image
FROM golang:1.19.0 as tilt-helper
Expand Down Expand Up @@ -312,7 +291,6 @@ def get_port_forwards(debug):
# 2. Configures a docker build for the provider, with live updating of the manager binary
# 3. Runs kustomize for the provider's config/default and applies it
def enable_provider(name, debug):
deployment_kind = "provider"
p = providers.get(name)
label = p.get("label")

Expand All @@ -336,77 +314,26 @@ def enable_provider(name, debug):
port_forwards = port_forwards,
)

additional_objs = []
p_resources = p.get("additional_resources", [])
for resource in p_resources:
k8s_yaml(p.get("context") + "/" + resource)
additional_objs = additional_objs + decode_yaml_stream(read_file(p.get("context") + "/" + resource))

if p.get("kustomize_config", True):
yaml = read_file("./.tiltbuild/yaml/{}.{}.yaml".format(name, deployment_kind))
yaml = read_file("./.tiltbuild/yaml/{}.provider.yaml".format(name))
k8s_yaml(yaml)
objs = decode_yaml_stream(yaml)
k8s_resource(
workload = find_object_name(objs, "Deployment"),
objects = [find_object_qualified_name(objs, "Provider")],
objects = [find_object_qualified_name(objs, "Provider")] + find_all_objects_names(additional_objs),
new_name = label.lower() + "_controller",
labels = [label, "ALL.controllers"],
port_forwards = port_forwards,
links = links,
resource_deps = ["provider_crd"],
)

# Configures an addon by doing the following:
#
# 1. Enables a local_resource go build of the addon's manager binary
# 2. Configures a docker build for the addon, with live updating of the binary
# 3. Runs kustomize for the addons's config/default and applies it
def enable_addon(name, debug):
addon = addons.get(name)
deployment_kind = "addon"
label = addon.get("label")
port_forwards, links = get_port_forwards(debug)

build_go_binary(
context = addon.get("context"),
reload_deps = addon.get("live_reload_deps"),
debug = debug,
go_main = addon.get("go_main", "main.go"),
binary_name = "extension",
label = label,
)

build_docker_image(
image = addon.get("image"),
context = addon.get("context"),
binary_name = "extension",
additional_docker_helper_commands = addon.get("additional_docker_helper_commands", ""),
additional_docker_build_commands = addon.get("additional_docker_build_commands", ""),
port_forwards = port_forwards,
)

additional_objs = []
addon_resources = addon.get("additional_resources", [])
for resource in addon_resources:
k8s_yaml(addon.get("context") + "/" + resource)
additional_objs = additional_objs + decode_yaml_stream(read_file(addon.get("context") + "/" + resource))

if addon.get("kustomize_config", True):
yaml = read_file("./.tiltbuild/yaml/{}.{}.yaml".format(name, deployment_kind))
objs = decode_yaml_stream(yaml)
k8s_yaml(yaml)
k8s_resource(
workload = find_object_name(objs, "Deployment"),
new_name = label.lower() + "_addon",
labels = [label, "ALL.addons"],
port_forwards = port_forwards,
links = links,
objects = find_all_objects_names(additional_objs),
resource_deps = addon.get("resource_deps", {}),
resource_deps = ["provider_crd"] + p.get("resource_deps", []),
)

def enable_addons():
for name in get_addons():
enable_addon(name, settings.get("debug").get(name, {}))

def get_addons():
user_enable_addons = settings.get("enable_addons", [])
return {k: "" for k in user_enable_addons}.keys()

def find_object_name(objs, kind):
for o in objs:
if o["kind"] == kind:
Expand Down Expand Up @@ -634,8 +561,6 @@ include_user_tilt_files()

load_provider_tiltfiles()

load_addon_tiltfiles()

prepare_all()

deploy_provider_crds()
Expand All @@ -644,6 +569,4 @@ deploy_observability()

enable_providers()

enable_addons()

cluster_templates()
4 changes: 4 additions & 0 deletions cmd/clusterctl/api/v1alpha3/labels.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,10 @@ func ManifestLabel(name string, providerType ProviderType) string {
return fmt.Sprintf("control-plane-%s", name)
case InfrastructureProviderType:
return fmt.Sprintf("infrastructure-%s", name)
case IPAMProviderType:
return fmt.Sprintf("ipam-%s", name)
case RuntimeExtensionProviderType:
return fmt.Sprintf("runtime-extension-%s", name)
default:
return name
}
Expand Down
18 changes: 16 additions & 2 deletions cmd/clusterctl/api/v1alpha3/provider_type.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,7 +92,9 @@ func (p *Provider) GetProviderType() ProviderType {
CoreProviderType,
BootstrapProviderType,
InfrastructureProviderType,
ControlPlaneProviderType:
ControlPlaneProviderType,
IPAMProviderType,
RuntimeExtensionProviderType:
return t
default:
return ProviderTypeUnknown
Expand All @@ -118,6 +120,14 @@ const (
// control-plane capabilities.
ControlPlaneProviderType = ProviderType("ControlPlaneProvider")

// IPAMProviderType is the type associated with codebases that provide
// IPAM capabilities.
IPAMProviderType = ProviderType("IPAMProvider")

// RuntimeExtensionProviderType is the type associated with codebases that provide
// runtime extensions.
RuntimeExtensionProviderType = ProviderType("RuntimeExtensionProvider")

// ProviderTypeUnknown is used when the type is unknown.
ProviderTypeUnknown = ProviderType("")
)
Expand All @@ -133,8 +143,12 @@ func (p ProviderType) Order() int {
return 2
case InfrastructureProviderType:
return 3
default:
case IPAMProviderType:
return 4
case RuntimeExtensionProviderType:
return 5
default:
return 99
}
}

Expand Down
10 changes: 7 additions & 3 deletions cmd/clusterctl/client/config/providers_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -361,14 +361,18 @@ func validateProvider(r Provider) error {
case clusterctlv1.CoreProviderType,
clusterctlv1.BootstrapProviderType,
clusterctlv1.InfrastructureProviderType,
clusterctlv1.ControlPlaneProviderType:
clusterctlv1.ControlPlaneProviderType,
clusterctlv1.IPAMProviderType,
clusterctlv1.RuntimeExtensionProviderType:
break
default:
return errors.Errorf("invalid provider type. Allowed values are [%s, %s, %s, %s]",
return errors.Errorf("invalid provider type. Allowed values are [%s, %s, %s, %s, %s, %s]",
clusterctlv1.CoreProviderType,
clusterctlv1.BootstrapProviderType,
clusterctlv1.InfrastructureProviderType,
clusterctlv1.ControlPlaneProviderType)
clusterctlv1.ControlPlaneProviderType,
clusterctlv1.IPAMProviderType,
clusterctlv1.RuntimeExtensionProviderType)
}
return nil
}
27 changes: 20 additions & 7 deletions cmd/clusterctl/client/delete.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,21 +30,24 @@ type DeleteOptions struct {
// default rules for kubeconfig discovery will be used.
Kubeconfig Kubeconfig

// CoreProvider version (e.g. cluster-api:v1.1.5) to add to the management cluster. If unspecified, the
// cluster-api core provider's latest release is used.
// CoreProvider version (e.g. cluster-api:v1.1.5) to delete from the management cluster.
CoreProvider string

// BootstrapProviders and versions (e.g. kubeadm:v1.1.5) to add to the management cluster.
// If unspecified, the kubeadm bootstrap provider's latest release is used.
// BootstrapProviders and versions (e.g. kubeadm:v1.1.5) to delete from the management cluster.
BootstrapProviders []string

// InfrastructureProviders and versions (e.g. aws:v0.5.0) to add to the management cluster.
// InfrastructureProviders and versions (e.g. aws:v0.5.0) to delete from the management cluster.
InfrastructureProviders []string

// ControlPlaneProviders and versions (e.g. kubeadm:v1.1.5) to add to the management cluster.
// If unspecified, the kubeadm control plane provider latest release is used.
// ControlPlaneProviders and versions (e.g. kubeadm:v1.1.5) to delete from the management cluster.
ControlPlaneProviders []string

// IPAMProviders and versions (e.g. infoblox:v0.0.1) to delete from the management cluster.
IPAMProviders []string

// RuntimeExtensionProviders and versions (e.g. test:v0.0.1) to delete from the management cluster.
RuntimeExtensionProviders []string

// DeleteAll set for deletion of all the providers.
DeleteAll bool

Expand Down Expand Up @@ -109,6 +112,16 @@ func (c *clusterctlClient) Delete(options DeleteOptions) error {
return err
}

providers, err = appendProviders(providers, clusterctlv1.IPAMProviderType, options.IPAMProviders...)
if err != nil {
return err
}

providers, err = appendProviders(providers, clusterctlv1.RuntimeExtensionProviderType, options.RuntimeExtensionProviders...)
if err != nil {
return err
}

for _, provider := range providers {
// Try to detect the namespace where the provider lives
provider.Namespace, err = clusterClient.ProviderInventory().GetProviderNamespace(provider.ProviderName, provider.GetProviderType())
Expand Down
14 changes: 14 additions & 0 deletions cmd/clusterctl/client/init.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,12 @@ type InitOptions struct {
// If unspecified, the kubeadm control plane provider latest release is used.
ControlPlaneProviders []string

// IPAMProviders and versions (e.g. infoblox:v0.0.1) to add to the management cluster.
IPAMProviders []string

// RuntimeExtensionProviders and versions (e.g. test:v0.0.1) to add to the management cluster.
RuntimeExtensionProviders []string

// TargetNamespace defines the namespace where the providers should be deployed. If unspecified, each provider
// will be installed in a provider's default namespace.
TargetNamespace string
Expand Down Expand Up @@ -227,6 +233,14 @@ func (c *clusterctlClient) setupInstaller(cluster cluster.Client, options InitOp
return nil, err
}

if err := c.addToInstaller(addOptions, clusterctlv1.IPAMProviderType, options.IPAMProviders...); err != nil {
return nil, err
}

if err := c.addToInstaller(addOptions, clusterctlv1.RuntimeExtensionProviderType, options.RuntimeExtensionProviders...); err != nil {
return nil, err
}

return installer, nil
}

Expand Down
18 changes: 17 additions & 1 deletion cmd/clusterctl/client/upgrade.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,12 @@ type ApplyUpgradeOptions struct {
// InfrastructureProviders instance and versions (e.g. capa-system/aws:v0.5.0) to upgrade to. This field can be used as alternative to Contract.
InfrastructureProviders []string

// IPAMProviders instance and versions (e.g. ipam-system/infoblox:v0.0.1) to upgrade to. This field can be used as alternative to Contract.
IPAMProviders []string

// RuntimeExtensionProviders instance and versions (e.g. runtime-extension-system/test:v0.0.1) to upgrade to. This field can be used as alternative to Contract.
RuntimeExtensionProviders []string

// WaitProviders instructs the upgrade apply command to wait till the providers are successfully upgraded.
WaitProviders bool

Expand Down Expand Up @@ -155,7 +161,9 @@ func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error {
isCustomUpgrade := options.CoreProvider != "" ||
len(options.BootstrapProviders) > 0 ||
len(options.ControlPlaneProviders) > 0 ||
len(options.InfrastructureProviders) > 0
len(options.InfrastructureProviders) > 0 ||
len(options.IPAMProviders) > 0 ||
len(options.RuntimeExtensionProviders) > 0

opts := cluster.UpgradeOptions{
WaitProviders: options.WaitProviders,
Expand Down Expand Up @@ -185,6 +193,14 @@ func (c *clusterctlClient) ApplyUpgrade(options ApplyUpgradeOptions) error {
if err != nil {
return err
}
upgradeItems, err = addUpgradeItems(upgradeItems, clusterctlv1.IPAMProviderType, options.IPAMProviders...)
if err != nil {
return err
}
upgradeItems, err = addUpgradeItems(upgradeItems, clusterctlv1.RuntimeExtensionProviderType, options.RuntimeExtensionProviders...)
if err != nil {
return err
}

// Execute the upgrade using the custom upgrade items
return clusterClient.ProviderUpgrader().ApplyCustomPlan(opts, upgradeItems...)
Expand Down
Loading