diff --git a/README.md b/README.md index 7898125..df235be 100644 --- a/README.md +++ b/README.md @@ -55,14 +55,6 @@ Make sure you have `kubectl`. `export KUBECONFIG="${HOME}/.kube/kind-config-management"` -1. Install the cluster-api CRDs - - `capdctl crds | kubectl apply -f -` - -1. Run the capd & capi manager - - `capdctl capd -capd-image=/capd-manager:latest | kubectl apply -f -` - ### Create a worker cluster `kubectl apply -f examples/simple-cluster.yaml` diff --git a/actuators/actuators.go b/actuators/actuators.go index c1bd5dc..0efd7d5 100644 --- a/actuators/actuators.go +++ b/actuators/actuators.go @@ -17,6 +17,7 @@ limitations under the License. package actuators import ( + "bytes" "fmt" "io/ioutil" @@ -71,6 +72,22 @@ func kubeconfigToSecret(clusterName, namespace string) (*v1.Secret, error) { return nil, errors.WithStack(err) } + allNodes, err := nodes.List(fmt.Sprintf("label=%s=%s", constants.ClusterLabelKey, clusterName)) + if err != nil { + return nil, errors.WithStack(err) + } + + // This is necessary so the management cluster in a container can talk to another container. + // They share the same bridged network and the load balancer does respond on 6443 at its docker IP + // however, the *HOST* is listening on some random port (the one returned from the GetLoadBalancerHostAndPort). + lbip, _, err := actions.GetLoadBalancerHostAndPort(allNodes) + lines := bytes.Split(data, []byte("\n")) + for i, line := range lines { + if bytes.Contains(line, []byte("https://")) { + lines[i] = []byte(fmt.Sprintf(" server: https://%s:%d", lbip, 6443)) + } + } + // write it to a secret return &v1.Secret{ ObjectMeta: metav1.ObjectMeta{ @@ -80,7 +97,7 @@ func kubeconfigToSecret(clusterName, namespace string) (*v1.Secret, error) { }, Data: map[string][]byte{ // TODO pull in constant from cluster api - "value": data, + "value": bytes.Join(lines, []byte("\n")), }, }, nil } diff --git a/actuators/machine.go b/actuators/machine.go index 8c561f1..97f269b 100644 --- a/actuators/machine.go +++ b/actuators/machine.go @@ -23,7 +23,6 @@ import ( "time" "github.com/go-logr/logr" - apicorev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" corev1 "k8s.io/client-go/kubernetes/typed/core/v1" "sigs.k8s.io/cluster-api-provider-docker/kind/actions" @@ -81,14 +80,9 @@ func (m *Machine) Create(ctx context.Context, c *clusterv1.Cluster, machine *clu m.Log.Error(err, "Error adding control plane") return err } - nodeUID, err := actions.GetNodeRefUID(c.GetName(), controlPlaneNode.Name()) - if err != nil { - m.Log.Error(err, "Error getting node reference UID") - return err - } - providerID := providerID(controlPlaneNode.Name()) + providerID := actions.ProviderID(controlPlaneNode.Name()) machine.Spec.ProviderID = &providerID - return m.save(old, machine, getNodeRef(controlPlaneNode.Name(), nodeUID)) + return m.save(old, machine) } m.Log.Info("Creating a brand new cluster") @@ -107,15 +101,10 @@ func (m *Machine) Create(ctx context.Context, c *clusterv1.Cluster, machine *clu m.Log.Error(err, "Error creating control plane") return err } - nodeUID, err := actions.GetNodeRefUID(c.GetName(), controlPlaneNode.Name()) - if err != nil { - m.Log.Error(err, "Error getting node reference UID") - return err - } // set the machine's providerID - providerID := providerID(controlPlaneNode.Name()) + providerID := actions.ProviderID(controlPlaneNode.Name()) machine.Spec.ProviderID = &providerID - if err := m.save(old, machine, getNodeRef(controlPlaneNode.Name(), nodeUID)); err != nil { + if err := m.save(old, machine); err != nil { m.Log.Error(err, "Error setting machine's provider ID") return err } @@ -144,14 +133,9 @@ func (m *Machine) Create(ctx context.Context, c *clusterv1.Cluster, machine *clu m.Log.Error(err, "Error creating new worker node") return err } - providerID := providerID(worker.Name()) + providerID := actions.ProviderID(worker.Name()) machine.Spec.ProviderID = &providerID - nodeUID, err := actions.GetNodeRefUID(c.GetName(), worker.Name()) - if err != nil { - m.Log.Error(err, "Error getting node reference ID") - return err - } - return m.save(old, machine, getNodeRef(worker.Name(), nodeUID)) + return m.save(old, machine) } // Delete returns nil when the machine no longer exists or when a successful delete has happened. @@ -201,7 +185,7 @@ func (m *Machine) Exists(ctx context.Context, cluster *clusterv1.Cluster, machin } // patches the object and saves the status. -func (m *Machine) save(oldMachine, newMachine *clusterv1.Machine, noderef *apicorev1.ObjectReference) error { +func (m *Machine) save(oldMachine, newMachine *clusterv1.Machine) error { m.Log.Info("updating machine") p, err := patch.NewJSONPatch(oldMachine, newMachine) if err != nil { @@ -222,19 +206,9 @@ func (m *Machine) save(oldMachine, newMachine *clusterv1.Machine, noderef *apico } m.Log.Info("updated machine") } - // set the noderef after so we don't try and patch it in during the first update - newMachine.Status.NodeRef = noderef - if _, err := m.ClusterAPI.Machines(oldMachine.Namespace).UpdateStatus(newMachine); err != nil { - m.Log.Error(err, "Error setting node reference") - return err - } return nil } -func providerID(name string) string { - return fmt.Sprintf("docker:////%s", name) -} - // CAPIroleToKindRole converts a CAPI role to kind role // TODO there is a better way to do this. func CAPIroleToKindRole(CAPIRole string) string { @@ -243,12 +217,3 @@ func CAPIroleToKindRole(CAPIRole string) string { } return CAPIRole } - -func getNodeRef(name, uid string) *apicorev1.ObjectReference { - return &apicorev1.ObjectReference{ - Kind: "Node", - APIVersion: apicorev1.SchemeGroupVersion.String(), - Name: name, - UID: types.UID(uid), - } -} diff --git a/cmd/capd-manager/main.go b/cmd/capd-manager/main.go index f015a6e..f640c80 100644 --- a/cmd/capd-manager/main.go +++ b/cmd/capd-manager/main.go @@ -17,6 +17,7 @@ limitations under the License. package main import ( + "flag" "fmt" "time" @@ -35,6 +36,9 @@ import ( ) func main() { + flag.Set("v", "0") + flag.Parse() + cfg, err := config.GetConfig() if err != nil { panic(err) @@ -67,6 +71,7 @@ func main() { machineLogger := logger.Log{} machineLogger.Logger = klogr.New().WithName("[machine-actuator]") + machineActuator := actuators.Machine{ Core: k8sclientset.CoreV1(), ClusterAPI: cs.ClusterV1alpha1(), diff --git a/cmd/capdctl/main.go b/cmd/capdctl/main.go index cc00d4d..a321235 100644 --- a/cmd/capdctl/main.go +++ b/cmd/capdctl/main.go @@ -17,20 +17,26 @@ limitations under the License. package main import ( + "archive/tar" + "bytes" + "compress/gzip" "encoding/json" - "errors" "flag" "fmt" + "io" "io/ioutil" + "net/http" "os" - "sigs.k8s.io/cluster-api-provider-docker/kind/actions" - "sigs.k8s.io/kind/pkg/cluster/nodes" - "sigs.k8s.io/kind/pkg/container/cri" - "sigs.k8s.io/kind/pkg/exec" + "strings" "time" + "github.com/pkg/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/cluster-api-provider-docker/kind/actions" "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1" + "sigs.k8s.io/kind/pkg/cluster/nodes" + "sigs.k8s.io/kind/pkg/container/cri" + "sigs.k8s.io/kind/pkg/exec" ) // TODO: Generate the RBAC stuff from somewhere instead of copy pasta @@ -68,12 +74,9 @@ func (mo *machineDeyploymentOptions) initFlags(fs *flag.FlagSet) { func main() { setup := flag.NewFlagSet("setup", flag.ExitOnError) managementClusterName := setup.String("cluster-name", "management", "The name of the management cluster") - - // crds takes no args - - capd := flag.NewFlagSet("capd", flag.ExitOnError) - capdImage := capd.String("capd-image", "gcr.io/kubernetes1-226021/capd-manager:latest", "The capd manager image to run") - capiImage := capd.String("capi-image", "gcr.io/k8s-cluster-api/cluster-api-controller:0.1.3", "The capi manager image to run") + version := setup.String("capi-version", "v0.1.4", "The CRD versions to pull from CAPI") + capdImage := setup.String("capd-image", "gcr.io/kubernetes1-226021/capd-manager:latest", "The capd manager image to run") + capiImage := setup.String("capi-image", "", "This is normally left blank and filled in automatically. But this will override the generated image name.") controlPlane := flag.NewFlagSet("control-plane", flag.ExitOnError) controlPlaneOpts := new(machineOptions) @@ -102,12 +105,7 @@ func main() { switch os.Args[1] { case "setup": setup.Parse(os.Args[2:]) - makeManagementCluster(*managementClusterName) - case "crds": - printCRDs() - case "capd": - capd.Parse(os.Args[2:]) - printClusterAPIPlane(*capdImage, *capiImage) + makeManagementCluster(*managementClusterName, *version, *capdImage, *capiImage) case "control-plane": controlPlane.Parse(os.Args[2:]) fmt.Fprintf(os.Stdout, machineYAML(controlPlaneOpts)) @@ -247,8 +245,12 @@ func machineYAML(opts *machineOptions) string { return string(b) } -func makeManagementCluster(clusterName string) { +func makeManagementCluster(clusterName, capiVersion, capdImage, capiImageOverride string) { fmt.Println("Creating a brand new cluster") + capiImage := fmt.Sprintf("gcr.io/k8s-cluster-api/cluster-api-controller:%s", capiVersion) + if capiImageOverride != "" { + capiImage = capiImageOverride + } elb, err := actions.SetUpLoadBalancer(clusterName) if err != nil { panic(err) @@ -275,9 +277,17 @@ func makeManagementCluster(clusterName string) { panic(err) } defer os.Remove(f.Name()) + fmt.Println("Downloading the latest CRDs for CAPI version", capiVersion) + crds, err := getCRDs(capiVersion, capiImage) + if err != nil { + panic(err) + } fmt.Fprintln(f, crds) fmt.Fprintln(f, "---") - fmt.Fprintln(f, getCAPDPlane("gcr.io/kubernetes1-226021/capd-manager:latest", "gcr.io/k8s-cluster-api/cluster-api-controller:0.1.3")) + fmt.Fprintln(f, capdRBAC) + fmt.Fprintln(f, "---") + fmt.Fprintln(f, getCAPDPlane(capdImage)) + fmt.Println("Applying the control plane", f.Name()) cmd := exec.Command("kubectl", "apply", "-f", f.Name()) cmd.SetEnv(fmt.Sprintf("KUBECONFIG=%s/.kube/kind-config-%s", os.Getenv("HOME"), clusterName)) cmd.SetStdout(os.Stdout) @@ -287,16 +297,8 @@ func makeManagementCluster(clusterName string) { } } -func printCRDs() { - fmt.Fprintln(os.Stdout, crds) -} - -func printClusterAPIPlane(capdImage, capiImage string) { - fmt.Fprintln(os.Stdout, getCAPDPlane(capdImage, capiImage)) -} - -func getCAPDPlane(capdImage, capiImage string) string { - return fmt.Sprintf(capiPlane, capdImage, capiImage) +func getCAPDPlane(capdImage string) string { + return fmt.Sprintf(capiPlane, capdImage) } var capiPlane = ` @@ -307,13 +309,6 @@ metadata: controller-tools.k8s.io: "1.0" name: docker-provider-system --- -apiVersion: v1 -kind: Namespace -metadata: - labels: - controller-tools.k8s.io: "1.0" - name: cluster-api-system ---- apiVersion: apps/v1 kind: StatefulSet metadata: @@ -362,950 +357,76 @@ spec: - effect: NoExecute key: node.alpha.kubernetes.io/unreachable operator: Exists ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - labels: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - name: cluster-api-controller-manager - namespace: cluster-api-system -spec: - selector: - matchLabels: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - serviceName: cluster-api-controller-manager-service - template: - metadata: - labels: - control-plane: controller-manager - controller-tools.k8s.io: "1.0" - spec: - containers: - - command: - - /manager - image: %s - name: manager - tolerations: - - effect: NoSchedule - key: node-role.kubernetes.io/master - - key: CriticalAddonsOnly - operator: Exists - - effect: NoExecute - key: node.alpha.kubernetes.io/notReady - operator: Exists - - effect: NoExecute - key: node.alpha.kubernetes.io/unreachable - operator: Exists ` -// TODO generate the CRDs -var crds = ` -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - creationTimestamp: null - labels: - controller-tools.k8s.io: "1.0" - name: clusters.cluster.k8s.io -spec: - group: cluster.k8s.io - names: - kind: Cluster - plural: clusters - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - clusterNetwork: - description: Cluster network configuration - properties: - pods: - description: The network ranges from which Pod networks are allocated. - properties: - cidrBlocks: - items: - type: string - type: array - required: - - cidrBlocks - type: object - serviceDomain: - description: Domain name for services. - type: string - services: - description: The network ranges from which service VIPs are allocated. - properties: - cidrBlocks: - items: - type: string - type: array - required: - - cidrBlocks - type: object - required: - - services - - pods - - serviceDomain - type: object - providerSpec: - description: Provider-specific serialized configuration to use during - cluster creation. It is recommended that providers maintain their - own versioned API types that should be serialized/deserialized from - this field. - properties: - value: - description: Value is an inlined, serialized representation of the - resource configuration. It is recommended that providers maintain - their own versioned API types that should be serialized/deserialized - from this field, akin to component config. - type: object - valueFrom: - description: Source for the provider configuration. Cannot be used - if value is not empty. - properties: - machineClass: - description: The machine class from which the provider config - should be sourced. - properties: - provider: - description: Provider is the name of the cloud-provider - which MachineClass is intended for. - type: string - type: object - type: object - type: object - required: - - clusterNetwork - type: object - status: - properties: - apiEndpoints: - description: APIEndpoint represents the endpoint to communicate with - the IP. - items: - properties: - host: - description: The hostname on which the API server is serving. - type: string - port: - description: The port on which the API server is serving. - format: int64 - type: integer - required: - - host - - port - type: object - type: array - errorMessage: - description: If set, indicates that there is a problem reconciling the - state, and will be set to a descriptive error message. - type: string - errorReason: - description: If set, indicates that there is a problem reconciling the - state, and will be set to a token value suitable for programmatic - interpretation. - type: string - providerStatus: - description: Provider-specific status. It is recommended that providers - maintain their own versioned API types that should be serialized/deserialized - from this field. - type: object - type: object - version: v1alpha1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - creationTimestamp: null - labels: - controller-tools.k8s.io: "1.0" - name: machineclasses.cluster.k8s.io -spec: - group: cluster.k8s.io - names: - kind: MachineClass - plural: machineclasses - scope: Namespaced - validation: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - metadata: - type: object - providerSpec: - description: Provider-specific configuration to use during node creation. - type: object - required: - - providerSpec - version: v1alpha1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - creationTimestamp: null - labels: - controller-tools.k8s.io: "1.0" - name: machinedeployments.cluster.k8s.io -spec: - group: cluster.k8s.io - names: - kind: MachineDeployment - plural: machinedeployments - scope: Namespaced - subresources: - scale: - labelSelectorPath: .status.labelSelector - specReplicasPath: .spec.replicas - statusReplicasPath: .status.replicas - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - minReadySeconds: - description: Minimum number of seconds for which a newly created machine - should be ready. Defaults to 0 (machine will be considered available - as soon as it is ready) - format: int32 - type: integer - paused: - description: Indicates that the deployment is paused. - type: boolean - progressDeadlineSeconds: - description: The maximum time in seconds for a deployment to make progress - before it is considered to be failed. The deployment controller will - continue to process failed deployments and a condition with a ProgressDeadlineExceeded - reason will be surfaced in the deployment status. Note that progress - will not be estimated during the time a deployment is paused. Defaults - to 600s. - format: int32 - type: integer - replicas: - description: Number of desired machines. Defaults to 1. This is a pointer - to distinguish between explicit zero and not specified. - format: int32 - type: integer - revisionHistoryLimit: - description: The number of old MachineSets to retain to allow rollback. - This is a pointer to distinguish between explicit zero and not specified. - Defaults to 1. - format: int32 - type: integer - selector: - description: Label selector for machines. Existing MachineSets whose - machines are selected by this will be the ones affected by this deployment. - It must match the machine template's labels. - type: object - strategy: - description: The deployment strategy to use to replace existing machines - with new ones. - properties: - rollingUpdate: - description: Rolling update config params. Present only if MachineDeploymentStrategyType - = RollingUpdate. - properties: - maxSurge: - description: 'The maximum number of machines that can be scheduled - above the desired number of machines. Value can be an absolute - number (ex: 5) or a percentage of desired machines (ex: 10%). - This can not be 0 if MaxUnavailable is 0. Absolute number - is calculated from percentage by rounding up. Defaults to - 1. Example: when this is set to 30%, the new MachineSet can - be scaled up immediately when the rolling update starts, such - that the total number of old and new machines do not exceed - 130% of desired machines. Once old machines have been killed, - new MachineSet can be scaled up further, ensuring that total - number of machines running at any time during the update is - at most 130% of desired machines.' - oneOf: - - type: string - - type: integer - maxUnavailable: - description: 'The maximum number of machines that can be unavailable - during the update. Value can be an absolute number (ex: 5) - or a percentage of desired machines (ex: 10%). Absolute number - is calculated from percentage by rounding down. This can not - be 0 if MaxSurge is 0. Defaults to 0. Example: when this is - set to 30%, the old MachineSet can be scaled down to 70% of - desired machines immediately when the rolling update starts. - Once new machines are ready, old MachineSet can be scaled - down further, followed by scaling up the new MachineSet, ensuring - that the total number of machines available at all times during - the update is at least 70% of desired machines.' - oneOf: - - type: string - - type: integer - type: object - type: - description: Type of deployment. Currently the only supported strategy - is "RollingUpdate". Default is RollingUpdate. - type: string - type: object - template: - description: Template describes the machines that will be created. - properties: - metadata: - description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - type: object - spec: - description: 'Specification of the desired behavior of the machine. - More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status' - properties: - configSource: - description: ConfigSource is used to populate in the associated - Node for dynamic kubelet config. This field already exists - in Node, so any updates to it in the Machine spec will be - automatically copied to the linked NodeRef from the status. - The rest of dynamic kubelet config support should then work - as-is. - type: object - metadata: - description: ObjectMeta will autopopulate the Node created. - Use this to indicate what labels, annotations, name prefix, - etc., should be used when creating the Node. - type: object - providerID: - description: ProviderID is the identification ID of the machine - provided by the provider. This field must match the provider - ID as seen on the node object corresponding to this machine. - This field is required by higher level consumers of cluster-api. - Example use case is cluster autoscaler with cluster-api as - provider. Clean-up logic in the autoscaler compares machines - to nodes to find out machines at provider which could not - get registered as Kubernetes nodes. With cluster-api as a - generic out-of-tree provider for autoscaler, this field is - required by autoscaler to be able to have a provider view - of the list of machines. Another list of nodes is queried - from the k8s apiserver and then a comparison is done to find - out unregistered machines and are marked for delete. This - field will be set by the actuators and consumed by higher - level entities like autoscaler that will be interfacing with - cluster-api as generic provider. - type: string - providerSpec: - description: ProviderSpec details Provider-specific configuration - to use during node creation. - properties: - value: - description: Value is an inlined, serialized representation - of the resource configuration. It is recommended that - providers maintain their own versioned API types that - should be serialized/deserialized from this field, akin - to component config. - type: object - valueFrom: - description: Source for the provider configuration. Cannot - be used if value is not empty. - properties: - machineClass: - description: The machine class from which the provider - config should be sourced. - properties: - provider: - description: Provider is the name of the cloud-provider - which MachineClass is intended for. - type: string - type: object - type: object - type: object - taints: - description: Taints is the full, authoritative list of taints - to apply to the corresponding Node. This list will overwrite - any modifications made to the Node on an ongoing basis. - items: - type: object - type: array - versions: - description: Versions of key software to use. This field is - optional at cluster creation time, and omitting the field - indicates that the cluster installation tool should select - defaults for the user. These defaults may differ based on - the cluster installer, but the tool should populate the values - it uses when persisting Machine objects. A Machine spec missing - this field at runtime is invalid. - properties: - controlPlane: - description: ControlPlane is the semantic version of the - Kubernetes control plane to run. This should only be populated - when the machine is a control plane. - type: string - kubelet: - description: Kubelet is the semantic version of kubelet - to run - type: string - required: - - kubelet - type: object - required: - - providerSpec - type: object - type: object - required: - - selector - - template - type: object - status: - properties: - availableReplicas: - description: Total number of available machines (ready for at least - minReadySeconds) targeted by this deployment. - format: int32 - type: integer - observedGeneration: - description: The generation observed by the deployment controller. - format: int64 - type: integer - readyReplicas: - description: Total number of ready machines targeted by this deployment. - format: int32 - type: integer - replicas: - description: Total number of non-terminated machines targeted by this - deployment (their labels match the selector). - format: int32 - type: integer - unavailableReplicas: - description: Total number of unavailable machines targeted by this deployment. - This is the total number of machines that are still required for the - deployment to have 100% available capacity. They may either be machines - that are running but not yet available or machines that still have - not been created. - format: int32 - type: integer - updatedReplicas: - description: Total number of non-terminated machines targeted by this - deployment that have the desired template spec. - format: int32 - type: integer - type: object - version: v1alpha1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - creationTimestamp: null - labels: - controller-tools.k8s.io: "1.0" - name: machines.cluster.k8s.io -spec: - additionalPrinterColumns: - - JSONPath: .spec.providerID - description: Provider ID - name: ProviderID - type: string - - JSONPath: .status.phase - description: Machine status such as Terminating/Pending/Running/Failed etc - name: Phase - type: string - group: cluster.k8s.io - names: - kind: Machine - plural: machines - scope: Namespaced - subresources: - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - configSource: - description: ConfigSource is used to populate in the associated Node - for dynamic kubelet config. This field already exists in Node, so - any updates to it in the Machine spec will be automatically copied - to the linked NodeRef from the status. The rest of dynamic kubelet - config support should then work as-is. - type: object - metadata: - description: ObjectMeta will autopopulate the Node created. Use this - to indicate what labels, annotations, name prefix, etc., should be - used when creating the Node. - type: object - providerID: - description: ProviderID is the identification ID of the machine provided - by the provider. This field must match the provider ID as seen on - the node object corresponding to this machine. This field is required - by higher level consumers of cluster-api. Example use case is cluster - autoscaler with cluster-api as provider. Clean-up logic in the autoscaler - compares machines to nodes to find out machines at provider which - could not get registered as Kubernetes nodes. With cluster-api as - a generic out-of-tree provider for autoscaler, this field is required - by autoscaler to be able to have a provider view of the list of machines. - Another list of nodes is queried from the k8s apiserver and then comparison - is done to find out unregistered machines and are marked for delete. - This field will be set by the actuators and consumed by higher level - entities like autoscaler that will be interfacing with cluster-api - as generic provider. - type: string - providerSpec: - description: ProviderSpec details Provider-specific configuration to - use during node creation. - properties: - value: - description: Value is an inlined, serialized representation of the - resource configuration. It is recommended that providers maintain - their own versioned API types that should be serialized/deserialized - from this field, akin to component config. - type: object - valueFrom: - description: Source for the provider configuration. Cannot be used - if value is not empty. - properties: - machineClass: - description: The machine class from which the provider config - should be sourced. - properties: - provider: - description: Provider is the name of the cloud-provider - which MachineClass is intended for. - type: string - type: object - type: object - type: object - taints: - description: Taints is the full, authoritative list of taints to apply - to the corresponding Node. This list will overwrite any modifications - made to the Node on an ongoing basis. - items: - type: object - type: array - versions: - description: Versions of key software to use. This field is optional - at cluster creation time, and omitting the field indicates that the - cluster installation tool should select defaults for the user. These - defaults may differ based on the cluster installer, but the tool should - populate the values it uses when persisting Machine objects. A Machine - spec missing this field at runtime is invalid. - properties: - controlPlane: - description: ControlPlane is the semantic version of the Kubernetes - control plane to run. This should only be populated when the machine - is a control plane. - type: string - kubelet: - description: Kubelet is the semantic version of kubelet to run - type: string - required: - - kubelet - type: object - required: - - providerSpec - type: object - status: - properties: - addresses: - description: Addresses is a list of addresses assigned to the machine. - Queried from cloud provider, if available. - items: - type: object - type: array - conditions: - description: 'Conditions lists the conditions synced from the node conditions - of the corresponding node-object. Machine-controller is responsible - for keeping conditions up-to-date. MachineSet controller will be taking - these conditions as a signal to decide if machine is healthy or needs - to be replaced. Refer: https://kubernetes.io/docs/concepts/architecture/nodes/#condition' - items: - type: object - type: array - errorMessage: - description: ErrorMessage will be set in the event that there is a terminal - problem reconciling the Machine and will contain a more verbose string - suitable for logging and human consumption. This field should not - be set for transitive errors that a controller faces that are expected - to be fixed automatically over time (like service outages), but instead - indicate that something is fundamentally wrong with the Machine's - spec or the configuration of the controller, and that manual intervention - is required. Examples of terminal errors would be invalid combinations - of settings in the spec, values that are unsupported by the controller, - or the responsible controller itself being critically misconfigured. Any - transient errors that occur during the reconciliation of Machines - can be added as events to the Machine object and/or logged in the - controller's output. - type: string - errorReason: - description: ErrorReason will be set in the event that there is a terminal - problem reconciling the Machine and will contain a succinct value - suitable for machine interpretation. This field should not be set - for transitive errors that a controller faces that are expected to - be fixed automatically over time (like service outages), but instead - indicate that something is fundamentally wrong with the Machine's - spec or the configuration of the controller, and that manual intervention - is required. Examples of terminal errors would be invalid combinations - of settings in the spec, values that are unsupported by the controller, - or the responsible controller itself being critically misconfigured. Any - transient errors that occur during the reconciliation of Machines - can be added as events to the Machine object and/or logged in the - controller's output. - type: string - lastOperation: - description: LastOperation describes the last-operation performed by - the machine-controller. This API should be useful as a history in - terms of the latest operation performed on the specific machine. It - should also convey the state of the latest-operation for example if - it is still on-going, failed or completed successfully. - properties: - description: - description: Description is the human-readable description of the - last operation. - type: string - lastUpdated: - description: LastUpdated is the timestamp at which LastOperation - API was last-updated. - format: date-time - type: string - state: - description: State is the current status of the last performed operation. - E.g. Processing, Failed, Successful etc - type: string - type: - description: Type is the type of operation which was last performed. - E.g. Create, Delete, Update etc - type: string - type: object - lastUpdated: - description: LastUpdated identifies when this status was last observed. - format: date-time - type: string - nodeRef: - description: NodeRef will point to the corresponding Node if it exists. - type: object - phase: - description: Phase represents the current phase of machine actuation. - E.g. Pending, Running, Terminating, Failed etc. - type: string - providerStatus: - description: ProviderStatus details a Provider-specific status. It is - recommended that providers maintain their own versioned API types - that should be serialized/deserialized from this field. - type: object - versions: - description: 'Versions specifies the current versions of software on - the corresponding Node (if it exists). This is provided for a few - reasons: 1) It is more convenient than checking the NodeRef, traversing - it to the Node, and finding the appropriate field in Node.Status.NodeInfo (which - uses different field names and formatting). 2) It removes some of - the dependency on the structure of the Node, so that if the structure - of Node.Status.NodeInfo changes, only machine controllers need - to be updated, rather than every client of the Machines API. 3) - There is no other simple way to check the control plane version. - A client would have to connect directly to the apiserver running - on the target node in order to find out its version.' - properties: - controlPlane: - description: ControlPlane is the semantic version of the Kubernetes - control plane to run. This should only be populated when the machine - is a control plane. - type: string - kubelet: - description: Kubelet is the semantic version of kubelet to run - type: string - required: - - kubelet - type: object - type: object - version: v1alpha1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: apiextensions.k8s.io/v1beta1 -kind: CustomResourceDefinition -metadata: - creationTimestamp: null - labels: - controller-tools.k8s.io: "1.0" - name: machinesets.cluster.k8s.io -spec: - group: cluster.k8s.io - names: - kind: MachineSet - plural: machinesets - scope: Namespaced - subresources: - scale: - labelSelectorPath: .status.labelSelector - specReplicasPath: .spec.replicas - statusReplicasPath: .status.replicas - status: {} - validation: - openAPIV3Schema: - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - properties: - deletePolicy: - description: DeletePolicy defines the policy used to identify nodes - to delete when downscaling. Defaults to "Random". Valid values are - "Random, "Newest", "Oldest" - enum: - - Random - - Newest - - Oldest - type: string - minReadySeconds: - description: MinReadySeconds is the minimum number of seconds for which - a newly created machine should be ready. Defaults to 0 (machine will - be considered available as soon as it is ready) - format: int32 - type: integer - replicas: - description: Replicas is the number of desired replicas. This is a pointer - to distinguish between explicit zero and unspecified. Defaults to - 1. - format: int32 - type: integer - selector: - description: 'Selector is a label query over machines that should match - the replica count. Label keys and values that must match in order - to be controlled by this MachineSet. It must match the machine template''s - labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors' - type: object - template: - description: Template is the object that describes the machine that - will be created if insufficient replicas are detected. - properties: - metadata: - description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#metadata' - type: object - spec: - description: 'Specification of the desired behavior of the machine. - More info: https://git.k8s.io/community/contributors/devel/api-conventions.md#spec-and-status' - properties: - configSource: - description: ConfigSource is used to populate in the associated - Node for dynamic kubelet config. This field already exists - in Node, so any updates to it in the Machine spec will be - automatically copied to the linked NodeRef from the status. - The rest of dynamic kubelet config support should then work - as-is. - type: object - metadata: - description: ObjectMeta will autopopulate the Node created. - Use this to indicate what labels, annotations, name prefix, - etc., should be used when creating the Node. - type: object - providerID: - description: ProviderID is the identification ID of the machine - provided by the provider. This field must match the provider - ID as seen on the node object corresponding to this machine. - This field is required by higher level consumers of cluster-api. - Example use case is cluster autoscaler with cluster-api as - provider. Clean-up logic in the autoscaler compares machines - to nodes to find out machines at provider which could not - get registered as Kubernetes nodes. With cluster-api as a - generic out-of-tree provider for autoscaler, this field is - required by autoscaler to be able to have a provider view - of the list of machines. Another list of nodes is queried - from the k8s apiserver and then a comparison is done to find - out unregistered machines and are marked for delete. This - field will be set by the actuators and consumed by higher - level entities like autoscaler that will be interfacing with - cluster-api as generic provider. - type: string - providerSpec: - description: ProviderSpec details Provider-specific configuration - to use during node creation. - properties: - value: - description: Value is an inlined, serialized representation - of the resource configuration. It is recommended that - providers maintain their own versioned API types that - should be serialized/deserialized from this field, akin - to component config. - type: object - valueFrom: - description: Source for the provider configuration. Cannot - be used if value is not empty. - properties: - machineClass: - description: The machine class from which the provider - config should be sourced. - properties: - provider: - description: Provider is the name of the cloud-provider - which MachineClass is intended for. - type: string - type: object - type: object - type: object - taints: - description: Taints is the full, authoritative list of taints - to apply to the corresponding Node. This list will overwrite - any modifications made to the Node on an ongoing basis. - items: - type: object - type: array - versions: - description: Versions of key software to use. This field is - optional at cluster creation time, and omitting the field - indicates that the cluster installation tool should select - defaults for the user. These defaults may differ based on - the cluster installer, but the tool should populate the values - it uses when persisting Machine objects. A Machine spec missing - this field at runtime is invalid. - properties: - controlPlane: - description: ControlPlane is the semantic version of the - Kubernetes control plane to run. This should only be populated - when the machine is a control plane. - type: string - kubelet: - description: Kubelet is the semantic version of kubelet - to run - type: string - required: - - kubelet - type: object - required: - - providerSpec - type: object - type: object - required: - - selector - type: object - status: - properties: - availableReplicas: - description: The number of available replicas (ready for at least minReadySeconds) - for this MachineSet. - format: int32 - type: integer - errorMessage: - type: string - errorReason: - description: In the event that there is a terminal problem reconciling - the replicas, both ErrorReason and ErrorMessage will be set. ErrorReason - will be populated with a succinct value suitable for machine interpretation, - while ErrorMessage will contain a more verbose string suitable for - logging and human consumption. These fields should not be set for - transitive errors that a controller faces that are expected to be - fixed automatically over time (like service outages), but instead - indicate that something is fundamentally wrong with the MachineTemplate's - spec or the configuration of the machine controller, and that manual - intervention is required. Examples of terminal errors would be invalid - combinations of settings in the spec, values that are unsupported - by the machine controller, or the responsible machine controller itself - being critically misconfigured. Any transient errors that occur during - the reconciliation of Machines can be added as events to the MachineSet - object and/or logged in the controller's output. - type: string - fullyLabeledReplicas: - description: The number of replicas that have labels matching the labels - of the machine template of the MachineSet. - format: int32 - type: integer - observedGeneration: - description: ObservedGeneration reflects the generation of the most - recently observed MachineSet. - format: int64 - type: integer - readyReplicas: - description: The number of ready replicas for this MachineSet. A machine - is considered ready when the node has been created and is "Ready". - format: int32 - type: integer - replicas: - description: Replicas is the most recently observed number of replicas. - format: int32 - type: integer - required: - - replicas - type: object - version: v1alpha1 -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] ---- -apiVersion: rbac.authorization.k8s.io/v1 +// getCRDs should actually use kustomize to correctly build the manager yaml. +// HACK: this is a hacked function +func getCRDs(version, capiImage string) (string, error) { + crds := []string{"crds", "rbac", "manager"} + releaseCode := fmt.Sprintf("https://github.com/kubernetes-sigs/cluster-api/archive/%s.tar.gz", version) + + resp, err := http.Get(releaseCode) + if err != nil { + return "", errors.WithStack(err) + } + + gz, err := gzip.NewReader(resp.Body) + if err != nil { + return "", errors.WithStack(err) + } + + tgz := tar.NewReader(gz) + var buf bytes.Buffer + + for { + header, err := tgz.Next() + + if err == io.EOF { + break + } + + if err != nil { + return "", errors.WithStack(err) + } + + switch header.Typeflag { + case tar.TypeDir: + continue + case tar.TypeReg: + for _, crd := range crds { + // Skip the kustomization files for now. Would like to use kustomize in future + if strings.HasSuffix(header.Name, "kustomization.yaml") { + continue + } + + // This is a poor person's kustomize + if strings.HasSuffix(header.Name, "manager.yaml") { + var managerBuf bytes.Buffer + io.Copy(&managerBuf, tgz) + lines := strings.Split(managerBuf.String(), "\n") + for _, line := range lines { + if strings.Contains(line, "image:") { + buf.WriteString(strings.Replace(line, "image: controller:latest", fmt.Sprintf("image: %s", capiImage), 1)) + buf.WriteString("\n") + continue + } + buf.WriteString(line) + buf.WriteString("\n") + } + } + + // These files don't need kustomize at all. + if strings.Contains(header.Name, fmt.Sprintf("config/%s/", crd)) { + io.Copy(&buf, tgz) + fmt.Fprintln(&buf, "---") + } + } + } + } + return buf.String(), nil +} + +var capdRBAC = `apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole metadata: name: docker-provider-manager-role @@ -1366,93 +487,6 @@ rules: - delete --- apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: cluster-api-manager-role -rules: -- apiGroups: - - "" - resources: - - events - verbs: - - get - - list - - watch - - create - - patch -- apiGroups: - - "" - resources: - - secrets - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - nodes - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - cluster.k8s.io - resources: - - clusters - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - cluster.k8s.io - resources: - - machines - - machines/status - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - cluster.k8s.io - resources: - - machinedeployments - - machinedeployments/status - verbs: - - get - - list - - watch - - create - - update - - patch - - delete -- apiGroups: - - cluster.k8s.io - resources: - - machinesets - - machinesets/status - verbs: - - get - - list - - watch - - create - - update - - patch - - delete ---- -apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: creationTimestamp: null @@ -1465,18 +499,4 @@ subjects: - kind: ServiceAccount name: default namespace: docker-provider-system ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - creationTimestamp: null - name: cluster-api-manager-rolebinding -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: cluster-api-manager-role -subjects: -- kind: ServiceAccount - name: default - namespace: cluster-api-system ` diff --git a/kind/actions/cluster_actions.go b/kind/actions/cluster_actions.go index 042b39d..9b9176c 100644 --- a/kind/actions/cluster_actions.go +++ b/kind/actions/cluster_actions.go @@ -143,7 +143,7 @@ func KubeadmConfig(node *nodes.Node, clusterName, lbip string) error { func KubeadmInit(clusterName, version string) error { allNodes, err := nodes.List(fmt.Sprintf("label=%s=%s", constants.ClusterLabelKey, clusterName)) if err != nil { - return nil + return err } node, err := nodes.BootstrapControlPlaneNode(allNodes) @@ -179,7 +179,7 @@ func KubeadmInit(clusterName, version string) error { } // save the kubeconfig on the host with the loadbalancer endpoint - hostPort, err := getLoadBalancerPort(allNodes) + _, hostPort, err := GetLoadBalancerHostAndPort(allNodes) if err != nil { return errors.Wrap(err, "failed to get kubeconfig from node") } @@ -270,7 +270,7 @@ func SetNodeProviderRef(clusterName, nodeName string) error { return err } - patch := fmt.Sprintf(`{"spec": {"providerID": "docker://%s"}}`, nodeName) + patch := fmt.Sprintf(`{"spec": {"providerID": "%s"}}`, ProviderID(nodeName)) fmt.Println("trying to apply:", patch) cmd := node.Command( "kubectl", @@ -290,39 +290,6 @@ func SetNodeProviderRef(clusterName, nodeName string) error { return nil } -// GetNodeRefUID returns the node reference UID -func GetNodeRefUID(clusterName, nodeName string) (string, error) { - // k get nodes my-cluster-worker -o custom-columns=UID:.metadata.uid --no-headers - allNodes, err := nodes.List(fmt.Sprintf("label=%s=%s", constants.ClusterLabelKey, clusterName)) - if err != nil { - return "", err - } - - node, err := nodes.BootstrapControlPlaneNode(allNodes) - if err != nil { - return "", err - } - - patch := fmt.Sprintf(`{"spec": {"providerID": "docker://%s"}}`, nodeName) - fmt.Println("trying to apply:", patch) - cmd := node.Command( - "kubectl", - "--kubeconfig", "/etc/kubernetes/admin.conf", - "get", - "node", nodeName, - "--output=custom-columns=UID:.metadata.uid", - "--no-headers", - ) - lines, err := exec.CombinedOutputLines(cmd) - if err != nil { - for _, line := range lines { - fmt.Println(line) - } - return "", errors.Wrap(err, "failed get node ref UID") - } - return strings.TrimSpace(lines[0]), nil -} - // DeleteClusterNode will remove the kubernetes node from the list of nodes (during a kubectl get nodes). func DeleteClusterNode(clusterName, nodeName string) error { // get all control plane nodes @@ -379,3 +346,8 @@ func KubeadmReset(clusterName, nodeName string) error { return nil } + +// ProviderID formats the provider id needed to set on the node +func ProviderID(name string) string { + return fmt.Sprintf("docker:////%s", name) +} diff --git a/kind/actions/kind.go b/kind/actions/kind.go index 714e194..3264455 100644 --- a/kind/actions/kind.go +++ b/kind/actions/kind.go @@ -236,13 +236,18 @@ func ListControlPlanes(clusterName string) ([]nodes.Node, error) { fmt.Sprintf("label=%s=%s", constants.NodeRoleKey, constants.ControlPlaneNodeRoleValue)) } -// getLoadBalancerPort returns the port on the host on which the APIServer is exposed -func getLoadBalancerPort(allNodes []nodes.Node) (int32, error) { +// GetLoadBalancerHostAndPort returns the port on the host on which the APIServer is exposed +func GetLoadBalancerHostAndPort(allNodes []nodes.Node) (string, int32, error) { node, err := nodes.ExternalLoadBalancerNode(allNodes) if err != nil { - return 0, err + return "", 0, err } - return node.Ports(6443) + ipv4, _, err := node.IP() + if err != nil { + return "", 0, err + } + port, err := node.Ports(6443) + return ipv4, port, err } // matches kubeconfig server entry like: