Skip to content

Commit

Permalink
Prototype of named machines for vsphere
Browse files Browse the repository at this point in the history
  • Loading branch information
karan committed Apr 26, 2018
1 parent 70c282d commit 1dd16bc
Show file tree
Hide file tree
Showing 16 changed files with 557 additions and 125 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ COPY . .
RUN CGO_ENABLED=0 GOOS=linux go install -a -ldflags '-extldflags "-static"' sigs.k8s.io/cluster-api/cloud/terraform/cmd/terraform-machine-controller

# Final container
FROM alpine3.7
FROM golang:alpine3.7
RUN apk --no-cache add --update ca-certificates bash openssh
RUN echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
RUN echo "UserKnownHostsFile /dev/null" >> /etc/ssh/ssh_config
Expand Down
5 changes: 3 additions & 2 deletions cloud/terraform/cmd/terraform-machine-controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,8 @@ import (
)

var (
kubeadmToken = pflag.String("token", "", "Kubeadm token to use to join new machines")
kubeadmToken = pflag.String("token", "", "Kubeadm token to use to join new machines")
namedMachinesPath = pflag.String("namedmachines", "", "path to named machines yaml file")
)

func init() {
Expand All @@ -54,7 +55,7 @@ func main() {
glog.Fatalf("Could not create client for talking to the apiserver: %v", err)
}

actuator, err := terraform.NewMachineActuator(*kubeadmToken, client.ClusterV1alpha1().Machines(corev1.NamespaceDefault))
actuator, err := terraform.NewMachineActuator(*kubeadmToken, client.ClusterV1alpha1().Machines(corev1.NamespaceDefault), *namedMachinesPath)
if err != nil {
glog.Fatalf("Could not create Terraform machine actuator: %v", err)
}
Expand Down
6 changes: 6 additions & 0 deletions cloud/terraform/config/configtemplate.go
Original file line number Diff line number Diff line change
Expand Up @@ -139,11 +139,14 @@ spec:
mountPath: /root/.terraform.d
- name: sshkeys
mountPath: /root/.ssh
- name: named-machines
mountPath: /etc/named-machines
command:
- "./terraform-machine-controller"
args:
- --kubeconfig=/etc/kubernetes/admin.conf
- --token={{ .Token }}
- --namedmachines=/etc/named-machines/vsphere_named_machines.yaml
resources:
requests:
cpu: 200m
Expand All @@ -167,6 +170,9 @@ spec:
- name: sshkeys
hostPath:
path: /home/ubuntu/.ssh
- name: named-machines
configMap:
name: named-machines
---
apiVersion: apps/v1beta1
kind: StatefulSet
Expand Down
132 changes: 88 additions & 44 deletions cloud/terraform/machineactuator.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,9 +32,13 @@ import (

"github.com/golang/glog"

corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/serializer"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/cluster-api/cloud/terraform/namedmachines"
terraformconfig "sigs.k8s.io/cluster-api/cloud/terraform/terraformproviderconfig"
terraformconfigv1 "sigs.k8s.io/cluster-api/cloud/terraform/terraformproviderconfig/v1alpha1"
apierrors "sigs.k8s.io/cluster-api/errors"
Expand All @@ -44,36 +48,68 @@ import (
)

const (
MasterIpAnnotationKey = "master-ip"
MasterIpAnnotationKey = "master-ip"
TerraformConfigAnnotationKey = "tf-config"

// Filename in which named machines are saved using a ConfigMap (in master).
NamedMachinesFilename = "vsphere_named_machines.yaml"
)

type TerraformClient struct {
scheme *runtime.Scheme
codecFactory *serializer.CodecFactory
kubeadmToken string
machineClient client.MachineInterface
scheme *runtime.Scheme
codecFactory *serializer.CodecFactory
kubeadmToken string
machineClient client.MachineInterface
namedMachineWatch *namedmachines.ConfigWatch
}

func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterface) (*TerraformClient, error) {
func NewMachineActuator(kubeadmToken string, machineClient client.MachineInterface, namedMachinePath string) (*TerraformClient, error) {
scheme, codecFactory, err := terraformconfigv1.NewSchemeAndCodecs()
if err != nil {
return nil, err
}

var nmWatch *namedmachines.ConfigWatch
nmWatch, err = namedmachines.NewConfigWatch(namedMachinePath)
if err != nil {
glog.Errorf("error creating named machine config watch: %+v", err)
}
return &TerraformClient{
scheme: scheme,
codecFactory: codecFactory,
kubeadmToken: kubeadmToken,
machineClient: machineClient,
scheme: scheme,
codecFactory: codecFactory,
kubeadmToken: kubeadmToken,
machineClient: machineClient,
namedMachineWatch: nmWatch,
}, nil
}

func (tf *TerraformClient) CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine) error {
func (tf *TerraformClient) CreateMachineController(cluster *clusterv1.Cluster, initialMachines []*clusterv1.Machine, clientSet kubernetes.Clientset) error {
if err := CreateExtApiServerRoleBinding(); err != nil {
return err
}

// Create the named machines ConfigMap.
// After pivot-based bootstrap is done, the named machine should be a ConfigMap and this logic will be removed.
namedMachines, err := tf.namedMachineWatch.NamedMachines()

if err != nil {
return err
}
yaml, err := namedMachines.GetYaml()
if err != nil {
return err
}
nmConfigMap := corev1.ConfigMap{
ObjectMeta: metav1.ObjectMeta{Name: "named-machines"},
Data: map[string]string{
NamedMachinesFilename: yaml,
},
}
configMaps := clientSet.CoreV1().ConfigMaps(corev1.NamespaceDefault)
if _, err := configMaps.Create(&nmConfigMap); err != nil {
return err
}

if err := CreateApiServerAndController(tf.kubeadmToken); err != nil {
return err
}
Expand All @@ -93,11 +129,11 @@ func getHomeDir() (string, error) {
}

// Fallback to user's profile.
usr, err := user.Current()
if err != nil {
return "", err
}
return usr.HomeDir, nil
usr, err := user.Current()
if err != nil {
return "", err
}
return usr.HomeDir, nil
}

func (tf *TerraformClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Machine) error {
Expand Down Expand Up @@ -126,7 +162,15 @@ func (tf *TerraformClient) Create(cluster *clusterv1.Cluster, machine *clusterv1
// Save the config file and variables file to the machinePath
tfConfigPath := path.Join(machinePath, "terraform.tf")
tfVarsPath := path.Join(machinePath, "variables.tfvars")
if err := saveFile(config.TerraformConfig, tfConfigPath, 0644); err != nil {
namedMachines, err := tf.namedMachineWatch.NamedMachines()
if err != nil {
return err
}
matchedMachine, err := namedMachines.MatchMachine(config.TerraformMachine)
if err != nil {
return err
}
if err := saveFile(matchedMachine.MachineHcl, tfConfigPath, 0644); err != nil {
return err
}
if err := saveFile(strings.Join(config.TerraformVariables, "\n"), tfVarsPath, 0644); err != nil {
Expand Down Expand Up @@ -286,7 +330,7 @@ func runTerraformCmd(stdout bool, workingDir string, arg ...string) (bytes.Buffe
logFileName := fmt.Sprintf("/tmp/cluster-api-%s.log", util.RandomToken())
f, _ := os.Create(logFileName)
glog.Infof("Running terraform. Check for logs in %s", logFileName)
multiWriter := io.MultiWriter(&out, f)
multiWriter := io.MultiWriter(&out, f)
cmd.Stdout = multiWriter
}
cmd.Stdin = os.Stdin
Expand Down Expand Up @@ -325,7 +369,7 @@ func (tf *TerraformClient) GetIP(machine *clusterv1.Machine) (string, error) {
if machine.ObjectMeta.Annotations != nil {
if ip, ok := machine.ObjectMeta.Annotations[MasterIpAnnotationKey]; ok {
glog.Infof("Retuning IP from metadata %s", ip)
return ip, nil
return ip, nil
}
}

Expand Down Expand Up @@ -354,9 +398,9 @@ func (tf *TerraformClient) GetKubeConfig(master *clusterv1.Machine) (string, err
"ssh", "-i", "~/.ssh/vsphere_tmp",
fmt.Sprintf("ubuntu@%s", ip),
"echo STARTFILE; cat /etc/kubernetes/admin.conf")
cmd.Stdout = &out
cmd.Stderr = os.Stderr
cmd.Run()
cmd.Stdout = &out
cmd.Stderr = os.Stderr
cmd.Run()
result := strings.TrimSpace(out.String())
parts := strings.Split(result, "STARTFILE")
if len(parts) != 2 {
Expand Down Expand Up @@ -391,33 +435,33 @@ func (tf *TerraformClient) SetupRemoteMaster(master *clusterv1.Machine) error {
"-r",
path.Join(homedir, ".terraform.d"),
fmt.Sprintf("ubuntu@%s:~/", ip))
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Run()
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Run()

// TODO: Bake this into the controller image instead of this hacky thing.
glog.Infof("Copying the terraform binary to master.")
// TODO: Bake this into the controller image instead of this hacky thing.
glog.Infof("Copying the terraform binary to master.")
cmd = exec.Command(
// TODO: this is taking my private key and username for now.
"scp", "-i", "~/.ssh/vsphere_tmp",
// TODO: this should be a flag?
"-r", "/Users/karangoel/.gvm/pkgsets/go1.9.2/global/src/sigs.k8s.io/cluster-api/cloud/terraform/bin/",
fmt.Sprintf("ubuntu@%s:~/.terraform.d/", ip))
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Run()
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Run()

glog.Infof("Setting up terraform on remote master.")
glog.Infof("Setting up terraform on remote master.")
cmd = exec.Command(
// TODO: this is taking my private key and username for now.
"ssh", "-i", "~/.ssh/vsphere_tmp",
fmt.Sprintf("ubuntu@%s", ip),
fmt.Sprintf("source ~/.profile; cd ~/.terraform.d/kluster/machines/%s; ~/.terraform.d/terraform init; cp -r ~/.terraform.d/kluster/machines/%s/.terraform/plugins/* ~/.terraform.d/plugins/", machineName, machineName))
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Run()
cmd.Stdout = os.Stdout
cmd.Stderr = os.Stderr
cmd.Run()

return nil
return nil
}

func (tf *TerraformClient) updateAnnotations(machine *clusterv1.Machine, masterEndpointIp string) error {
Expand Down Expand Up @@ -534,12 +578,12 @@ func run(cmd string, args ...string) error {
}

func pathExists(path string) (bool, error) {
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
_, err := os.Stat(path)
if err == nil {
return true, nil
}
if os.IsNotExist(err) {
return false, nil
}
return true, err
}
94 changes: 94 additions & 0 deletions cloud/terraform/namedmachines/namedmachines.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package namedmachines

import (
"fmt"
"io"
"io/ioutil"
"os"

"github.com/ghodss/yaml"
)

// Config Watch holds the path to the named machines yaml file.
// This is used during bootstrap when the apiserver does not yet exist.
type ConfigWatch struct {
path string
}

// A single named machine.
type NamedMachine struct {
MachineName string
MachineHcl string
}

// A list of named machines.
type NamedMachinesItems struct {
Items []NamedMachine `json:"items"`
}

// All named machines defined in yaml.
type NamedMachines struct {
namedMachinesItems *NamedMachinesItems
}

func NewConfigWatch(path string) (*ConfigWatch, error) {
if _, err := os.Stat(path); err != nil {
return nil, err
}
return &ConfigWatch{path: path}, nil
}

// Returns all named machines for ConfigWatch.
func (cw *ConfigWatch) NamedMachines() (*NamedMachines, error) {
file, err := os.Open(cw.path)
if err != nil {
return nil, err
}
return parseNamedMachinesYaml(file)
}

func parseNamedMachinesYaml(reader io.Reader) (*NamedMachines, error) {
bytes, err := ioutil.ReadAll(reader)
if err != nil {
return nil, err
}

items := &NamedMachinesItems{}
err = yaml.Unmarshal(bytes, items)
if err != nil {
return nil, err
}

return &NamedMachines{items}, nil
}

func (nm *NamedMachines) GetYaml() (string, error) {
bytes, err := yaml.Marshal(nm.namedMachinesItems)
if err != nil {
return "", err
}
return string(bytes), nil
}

// Returns a NamedMachine that matches the passed name.
func (nm *NamedMachines) MatchMachine(machineName string) (*NamedMachine, error) {
for _, namedMachine := range nm.namedMachinesItems.Items {
if namedMachine.MachineName == machineName {
return &namedMachine, nil
}
}
return nil, fmt.Errorf("could not find a machine with name %s", machineName)
}
5 changes: 2 additions & 3 deletions cloud/terraform/terraformproviderconfig/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,9 +24,8 @@ import (
type TerraformProviderConfig struct {
metav1.TypeMeta `json:",inline"`

// Contents of a terrafrom config file.
// This is the HCL config encoded as a string.
TerraformConfig string `json:"terraformConfig"`
// Name of the machine that's registered in the NamedMachines ConfigMap.
TerraformMachine string `json:"terraformMachine"`
// List of contents of terraform variables used.
// HCL variables encoded as string.
TerraformVariables []string `json:"terraformVariables"`
Expand Down
4 changes: 2 additions & 2 deletions cloud/terraform/terraformproviderconfig/v1alpha1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,8 @@ import (
type TerraformProviderConfig struct {
metav1.TypeMeta `json:",inline"`

// Contents of a terrafrom config file.
TerraformConfig string `json:"terraformConfig"`
// Name of the machine that's registered in the NamedMachines ConfigMap.
TerraformMachine string `json:"terraformMachine"`
// List of contents of terraform variable files used.
TerraformVariables []string `json:"terraformVariables"`
}
Loading

0 comments on commit 1dd16bc

Please sign in to comment.