Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use kubenet instead of weavenet for GCP provider #107

Merged
merged 11 commits into from
Apr 30, 2018
Merged

Use kubenet instead of weavenet for GCP provider #107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
ebb8c52
Use kubenet instead of weave net
kcoronado Apr 23, 2018
e391bd4
Remove --ignore-preflight-errors=all flag
kcoronado Apr 23, 2018
ee2219e
Fix issue with assigning conflicting IPs to pods
kcoronado Apr 23, 2018
d03a96f
Add check to only add service account to masters
kcoronado Apr 26, 2018
d30c4f5
Use apt-get instead of apt in scripts
kcoronado Apr 26, 2018
af87544
Remove ServiceAccountActor role from machine controller
kcoronado Apr 27, 2018
0ab8252
Add worker tag to master to make it schedulable
kcoronado Apr 30, 2018
f705c9a
Add docker change to nodes to prevent auto-start
kcoronado Apr 30, 2018
fc93913
Update machines yaml template to use kubenet configs
kcoronado Apr 30, 2018
777f872
Revert support for load balancing
kcoronado Apr 30, 2018
78cd0e2
Add comment and remove extra enable/start docker in node script
kcoronado Apr 30, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 24 additions & 10 deletions cloud/google/machineactuator.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,13 +40,13 @@ import (

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/client-go/kubernetes"
"sigs.k8s.io/cluster-api/cloud/google/clients"
gceconfig "sigs.k8s.io/cluster-api/cloud/google/gceproviderconfig"
gceconfigv1 "sigs.k8s.io/cluster-api/cloud/google/gceproviderconfig/v1alpha1"
"sigs.k8s.io/cluster-api/cloud/google/machinesetup"
apierrors "sigs.k8s.io/cluster-api/errors"
clusterv1 "sigs.k8s.io/cluster-api/pkg/apis/cluster/v1alpha1"
client "sigs.k8s.io/cluster-api/pkg/client/clientset_generated/clientset/typed/cluster/v1alpha1"
"sigs.k8s.io/cluster-api/cloud/google/clients"
"sigs.k8s.io/cluster-api/util"
)

Expand Down Expand Up @@ -78,12 +78,12 @@ type GCEClientComputeService interface {

type GCEClient struct {
computeService GCEClientComputeService
scheme *runtime.Scheme
codecFactory *serializer.CodecFactory
kubeadmToken string
sshCreds SshCreds
machineClient client.MachineInterface
configWatch *machinesetup.ConfigWatch
scheme *runtime.Scheme
codecFactory *serializer.CodecFactory
kubeadmToken string
sshCreds SshCreds
machineClient client.MachineInterface
configWatch *machinesetup.ConfigWatch
}

const (
Expand Down Expand Up @@ -267,9 +267,22 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
labels[BootstrapLabelKey] = "true"
}

// The service account is needed for the Kubernetes GCE cloud provider code. It is needed on the master VM.
serviceAccounts := []*compute.ServiceAccount{nil}
if util.IsMaster(machine) {
serviceAccounts = append(serviceAccounts,
&compute.ServiceAccount{
Email: "default",
Scopes: []string{
"https://www.googleapis.com/auth/cloud-platform",
},
})
}

op, err := gce.computeService.InstancesInsert(project, zone, &compute.Instance{
Name: name,
MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", zone, config.MachineType),
Name: name,
MachineType: fmt.Sprintf("zones/%s/machineTypes/%s", zone, config.MachineType),
CanIpForward: true,
NetworkInterfaces: []*compute.NetworkInterface{
{
Network: "global/networks/default",
Expand Down Expand Up @@ -297,7 +310,8 @@ func (gce *GCEClient) Create(cluster *clusterv1.Cluster, machine *clusterv1.Mach
Tags: &compute.Tags{
Items: []string{"https-server"},
},
Labels: labels,
Labels: labels,
ServiceAccounts: serviceAccounts,
})

if err == nil {
Expand Down
204 changes: 39 additions & 165 deletions gcp-deployer/machine_setup_configs.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,160 +1,4 @@
items:
- machineParams:
- os: ubuntu-1710-weave
roles:
- Node
versions:
kubelet: 1.9.4
containerRuntime:
name: docker
version: 1.12.0
image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710
metadata:
startupScript: |
set -e
set -x
(
apt-get update
apt-get install -y apt-transport-https prips
apt-key adv --keyserver hkp://keyserver.ubuntu.com --recv-keys F76221572C52609D
cat <<EOF > /etc/apt/sources.list.d/k8s.list
deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main
EOF
apt-get update
apt-get install -y docker-engine=1.12.0-0~xenial
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
cat <<EOF > /etc/apt/sources.list.d/kubernetes.list
deb http://apt.kubernetes.io/ kubernetes-xenial main
EOF
apt-get update
# Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix
# search based on our SemVer to find the right (newest) package version.
function getversion() {
name=$1
prefix=$2
version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1)
if [[ -z "$version" ]]; then
echo Can\'t find package $name with prefix $prefix
exit 1
fi
echo $version
}
KUBELET=$(getversion kubelet ${KUBELET_VERSION}-)
KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-)
KUBECTL=$(getversion kubectl ${KUBELET_VERSION}-)
apt-get install -y kubelet=${KUBELET} kubeadm=${KUBEADM} kubectl=${KUBECTL}
systemctl enable docker || true
systemctl start docker || true
# kubeadm uses 10th IP as DNS server
CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1)
# Override Kubelet DNS args.
cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<EOF
[Service]
Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
EOF
systemctl daemon-reload
systemctl restart kubelet.service
kubeadm join --token "${TOKEN}" "${MASTER}" --ignore-preflight-errors=all --discovery-token-unsafe-skip-ca-verification
for tries in $(seq 1 60); do
kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
sleep 1
done
echo done.
) 2>&1 | tee /var/log/startup.log
- machineParams:
- os: ubuntu-1710-weave
roles:
- Master
versions:
kubelet: 1.9.4
controlPlane: 1.9.4
containerRuntime:
name: docker
version: 1.12.0
image: projects/ubuntu-os-cloud/global/images/family/ubuntu-1710
metadata:
startupScript: |
set -e
set -x
(
ARCH=amd64
curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key add -
touch /etc/apt/sources.list.d/kubernetes.list
sh -c 'echo "deb http://apt.kubernetes.io/ kubernetes-xenial main" > /etc/apt/sources.list.d/kubernetes.list'
apt-get update -y
apt-get install -y \
socat \
ebtables \
docker.io \
apt-transport-https \
cloud-utils \
prips
curl -sSL https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubeadm > /usr/bin/kubeadm.dl
chmod a+rx /usr/bin/kubeadm.dl
# kubeadm uses 10th IP as DNS server
CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1)
# Our Debian packages have versions like "1.8.0-00" or "1.8.0-01". Do a prefix
# search based on our SemVer to find the right (newest) package version.
function getversion() {
name=$1
prefix=$2
version=$(apt-cache madison $name | awk '{ print $3 }' | grep ^$prefix | head -n1)
if [[ -z "$version" ]]; then
echo Can\'t find package $name with prefix $prefix
exit 1
fi
echo $version
}
KUBELET=$(getversion kubelet ${KUBELET_VERSION}-)
KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-)
apt-get install -y \
kubelet=${KUBELET} \
kubeadm=${KUBEADM}
mv /usr/bin/kubeadm.dl /usr/bin/kubeadm
chmod a+rx /usr/bin/kubeadm
systemctl enable docker
systemctl start docker
# Override Kubelet DNS args.
cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<EOF
[Service]
Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
EOF
systemctl daemon-reload
systemctl restart kubelet.service
PRIVATEIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip"`
echo $PRIVATEIP > /tmp/.ip
PUBLICIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip"`
# Set up kubeadm config file to pass parameters to kubeadm init.
cat > /etc/kubernetes/kubeadm_config.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1alpha1
kind: MasterConfiguration
api:
advertiseAddress: ${PUBLICIP}
bindPort: ${PORT}
networking:
serviceSubnet: ${SERVICE_CIDR}
kubernetesVersion: v${CONTROL_PLANE_VERSION}
token: ${TOKEN}
controllerManagerExtraArgs:
cluster-cidr: ${POD_CIDR}
service-cluster-ip-range: ${SERVICE_CIDR}
allocate-node-cidrs: "true"
apiServerCertSANs:
- ${PUBLICIP}
- ${PRIVATEIP}
EOF
kubeadm init --config /etc/kubernetes/kubeadm_config.yaml
# install weavenet
sysctl net.bridge.bridge-nf-call-iptables=1
export kubever=$(kubectl version --kubeconfig /etc/kubernetes/admin.conf | base64 | tr -d '\n')
kubectl apply --kubeconfig /etc/kubernetes/admin.conf -f "https://cloud.weave.works/k8s/net?k8s-version=$kubever"
for tries in $(seq 1 60); do
kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
sleep 1
done
echo done.
) 2>&1 | tee /var/log/startup.log
# These configs currently don't work - they need service accounts.
- machineParams:
- os: ubuntu-1604-lts
roles:
Expand All @@ -179,10 +23,23 @@ items:
apt-get install -y \
socat \
ebtables \
docker.io \
apt-transport-https \
cloud-utils \
prips

function install_configure_docker () {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wouldn't similar changes be required for the nodes?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I added the change in for nodes.

# prevent docker from auto-starting
echo "exit 101" > /usr/sbin/policy-rc.d
chmod +x /usr/sbin/policy-rc.d
trap "rm /usr/sbin/policy-rc.d" RETURN
apt-get install -y docker.io
echo 'DOCKER_OPTS="--iptables=false --ip-masq=false"' > /etc/default/docker
systemctl daemon-reload
systemctl enable docker
systemctl start docker
}
install_configure_docker

curl -sSL https://dl.k8s.io/release/${VERSION}/bin/linux/${ARCH}/kubeadm > /usr/bin/kubeadm.dl
chmod a+rx /usr/bin/kubeadm.dl
# kubeadm uses 10th IP as DNS server
Expand All @@ -206,19 +63,19 @@ items:
kubeadm=${KUBEADM}
mv /usr/bin/kubeadm.dl /usr/bin/kubeadm
chmod a+rx /usr/bin/kubeadm
systemctl enable docker
systemctl start docker

# Override network args to use kubenet instead of cni, and override Kubelet DNS args.
cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<EOF
[Service]
Environment="KUBELET_NETWORK_ARGS=--network-plugin=kubenet --pod-cidr=${POD_CIDR}"
Environment="KUBELET_NETWORK_ARGS=--network-plugin=kubenet"
Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
EOF
systemctl daemon-reload
systemctl restart kubelet.service
PRIVATEIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/ip"`
echo $PRIVATEIP > /tmp/.ip
PUBLICIP=`curl --retry 5 -sfH "Metadata-Flavor: Google" "http://metadata/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip"`

# Set up the GCE cloud config, which gets picked up by kubeadm init since cloudProvider is set to GCE.
cat > /etc/kubernetes/cloud-config <<EOF
[global]
Expand All @@ -227,6 +84,7 @@ items:
subnetwork-name = ${SUBNETWORK}
node-tags = ${NODE_TAG}
EOF

# Set up kubeadm config file to pass parameters to kubeadm init.
cat > /etc/kubernetes/kubeadm_config.yaml <<EOF
apiVersion: kubeadm.k8s.io/v1alpha1
Expand All @@ -247,13 +105,18 @@ items:
- ${PUBLICIP}
- ${PRIVATEIP}
EOF

# Create and set bridge-nf-call-iptables to 1 to pass the kubeadm preflight check.
# Workaround was found here:
# http://zeeshanali.com/sysadmin/fixed-sysctl-cannot-stat-procsysnetbridgebridge-nf-call-iptables/
modprobe br_netfilter

kubeadm init --config /etc/kubernetes/kubeadm_config.yaml
for tries in $(seq 1 60); do
kubectl --kubeconfig /etc/kubernetes/kubelet.conf annotate --overwrite node $(hostname) machine=${MACHINE} && break
sleep 1
done
echo done.
# kubectl create configmap cloud-config --namespace=kube-system --from-file=./gce.conf
) 2>&1 | tee /var/log/startup.log
- machineParams:
- os: ubuntu-1604-lts
Expand All @@ -277,7 +140,20 @@ items:
deb [arch=amd64] https://apt.dockerproject.org/repo ubuntu-xenial main
EOF
apt-get update
apt-get install -y docker-engine=1.12.0-0~xenial

function install_configure_docker () {
# prevent docker from auto-starting
echo "exit 101" > /usr/sbin/policy-rc.d
chmod +x /usr/sbin/policy-rc.d
trap "rm /usr/sbin/policy-rc.d" RETURN
apt-get install -y docker-engine=1.12.0-0~xenial
echo 'DOCKER_OPTS="--iptables=false --ip-masq=false"' > /etc/default/docker
systemctl daemon-reload
systemctl enable docker
systemctl start docker
}
install_configure_docker

curl -s https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key add -
cat <<EOF > /etc/apt/sources.list.d/kubernetes.list
deb http://apt.kubernetes.io/ kubernetes-xenial main
Expand All @@ -299,14 +175,12 @@ items:
KUBEADM=$(getversion kubeadm ${KUBELET_VERSION}-)
KUBECTL=$(getversion kubectl ${KUBELET_VERSION}-)
apt-get install -y kubelet=${KUBELET} kubeadm=${KUBEADM} kubectl=${KUBECTL}
systemctl enable docker || true
systemctl start docker || true
# kubeadm uses 10th IP as DNS server
CLUSTER_DNS_SERVER=$(prips ${SERVICE_CIDR} | head -n 11 | tail -n 1)
# Override network args to use kubenet instead of cni, and override Kubelet DNS args.
cat > /etc/systemd/system/kubelet.service.d/20-kubenet.conf <<EOF
[Service]
Environment="KUBELET_NETWORK_ARGS=--network-plugin=kubenet --pod-cidr=${POD_CIDR}"
Environment="KUBELET_NETWORK_ARGS=--network-plugin=kubenet"
Environment="KUBELET_DNS_ARGS=--cluster-dns=${CLUSTER_DNS_SERVER} --cluster-domain=${CLUSTER_DNS_DOMAIN}"
EOF
systemctl daemon-reload
Expand Down
4 changes: 2 additions & 2 deletions gcp-deployer/machines.yaml.template
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ items:
project: "$GCLOUD_PROJECT"
zone: "$ZONE"
machineType: "n1-standard-2"
os: "ubuntu-1710-weave"
os: "ubuntu-1604-lts"
versions:
kubelet: 1.9.4
controlPlane: 1.9.4
Expand All @@ -36,7 +36,7 @@ items:
project: "$GCLOUD_PROJECT"
zone: "$ZONE"
machineType: "n1-standard-1"
os: "ubuntu-1710-weave"
os: "ubuntu-1604-lts"
versions:
kubelet: 1.9.4
containerRuntime:
Expand Down