From 03389c75a04fa0ecfcc8342d51adab641794e412 Mon Sep 17 00:00:00 2001 From: guohaowang Date: Thu, 10 Oct 2019 17:37:01 +0800 Subject: [PATCH 1/2] feature: support ha cluster with mutil-AZ Signed-off-by: guohaowang --- examples/aws-region-az.sh | 58 +++++++++++++++++++++++++ examples/cluster/cluster.yaml | 26 ++++++++++- examples/controlplane/controlplane.yaml | 3 ++ examples/generate.sh | 4 ++ hack/aws-region-az-fetch.sh | 28 ++++++++++++ 5 files changed, 118 insertions(+), 1 deletion(-) create mode 100755 examples/aws-region-az.sh create mode 100755 hack/aws-region-az-fetch.sh diff --git a/examples/aws-region-az.sh b/examples/aws-region-az.sh new file mode 100755 index 0000000000..a264f851f3 --- /dev/null +++ b/examples/aws-region-az.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +export AWS_REGION_AZA=${AWS_REGION}a +export AWS_REGION_AZB=${AWS_REGION}a +export AWS_REGION_AZC=${AWS_REGION}a + +REGIONMAPAZLIST=" +eu-north-1:eu-north-1a eu-north-1b eu-north-1c, +ap-south-1:ap-south-1a ap-south-1b ap-south-1c, +eu-west-3:eu-west-3a eu-west-3b eu-west-3c, +eu-west-2:eu-west-2a eu-west-2b eu-west-2c, +eu-west-1:eu-west-1a eu-west-1b eu-west-1c, +ap-northeast-2:ap-northeast-2a ap-northeast-2b ap-northeast-2c, +ap-northeast-1:ap-northeast-1a ap-northeast-1c ap-northeast-1d, +sa-east-1:sa-east-1a sa-east-1b sa-east-1c, +ca-central-1:ca-central-1a ca-central-1b, +ap-east-1:ap-east-1a ap-east-1b ap-east-1c, +ap-southeast-1:ap-southeast-1a ap-southeast-1b ap-southeast-1c, +ap-southeast-2:ap-southeast-2a ap-southeast-2b ap-southeast-2c, +eu-central-1:eu-central-1a eu-central-1b eu-central-1c, +us-east-1:us-east-1a us-east-1b us-east-1c us-east-1d us-east-1e us-east-1f, +us-east-2:us-east-2a us-east-2b us-east-2c, +us-west-1:us-west-1b us-west-1c, +us-west-2:us-west-2a us-west-2b us-west-2c us-west-2d, +" + +function set_region_az_info() { + declare -a AZLIST='('$(echo $REGIONMAPAZLIST | tr ',' '\n' | grep $1 | cut -d":" -f2)')' + if [ ${#AZLIST[@]} -lt 3 ]; + then + echo "this region not three AZ" + AWS_REGION_AZA=${AZLIST[0]} + AWS_REGION_AZB=${AZLIST[0]} + AWS_REGION_AZC=${AZLIST[0]} + else + AWS_REGION_AZA=${AZLIST[0]} + AWS_REGION_AZB=${AZLIST[1]} + AWS_REGION_AZC=${AZLIST[2]} + fi +} \ No newline at end of file diff --git a/examples/cluster/cluster.yaml b/examples/cluster/cluster.yaml index 8d95c2dfdf..7c1becb363 100644 --- a/examples/cluster/cluster.yaml +++ b/examples/cluster/cluster.yaml @@ -6,7 +6,11 @@ metadata: spec: clusterNetwork: pods: - cidrBlocks: ["192.168.0.0/16"] + cidrBlocks: + - "192.168.0.0/16" + services: + cidrBlocks: + - "192.168.1.0/24" infrastructureRef: apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 kind: AWSCluster @@ -19,3 +23,23 @@ metadata: spec: region: ${AWS_REGION} sshKeyName: ${SSH_KEY_NAME} + networkSpec: + subnets: + - availabilityZone: ${AWS_REGION_AZA} + cidrBlock: "10.0.100.0/24" + isPublic: true + - availabilityZone: ${AWS_REGION_AZA} + cidrBlock: "10.0.101.0/24" + isPublic: false + - availabilityZone: ${AWS_REGION_AZB} + cidrBlock: "10.0.102.0/24" + isPublic: true + - availabilityZone: ${AWS_REGION_AZB} + cidrBlock: "10.0.103.0/24" + isPublic: false + - availabilityZone: ${AWS_REGION_AZC} + cidrBlock: "10.0.104.0/24" + isPublic: true + - availabilityZone: ${AWS_REGION_AZC} + cidrBlock: "10.0.105.0/24" + isPublic: false \ No newline at end of file diff --git a/examples/controlplane/controlplane.yaml b/examples/controlplane/controlplane.yaml index 666df2ac53..7e46583c82 100644 --- a/examples/controlplane/controlplane.yaml +++ b/examples/controlplane/controlplane.yaml @@ -25,6 +25,7 @@ spec: instanceType: ${CONTROL_PLANE_MACHINE_TYPE} iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io" sshKeyName: "${SSH_KEY_NAME}" + availabilityZone: ${AWS_REGION_AZA} --- apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2 kind: KubeadmConfig @@ -71,6 +72,7 @@ spec: instanceType: ${CONTROL_PLANE_MACHINE_TYPE} iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io" sshKeyName: "${SSH_KEY_NAME}" + availabilityZone: ${AWS_REGION_AZB} --- apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2 kind: KubeadmConfig @@ -111,6 +113,7 @@ spec: instanceType: ${CONTROL_PLANE_MACHINE_TYPE} iamInstanceProfile: "control-plane.cluster-api-provider-aws.sigs.k8s.io" sshKeyName: "${SSH_KEY_NAME}" + availabilityZone: ${AWS_REGION_AZC} --- apiVersion: bootstrap.cluster.x-k8s.io/v1alpha2 kind: KubeadmConfig diff --git a/examples/generate.sh b/examples/generate.sh index 67f69f0944..0e3cd726f7 100755 --- a/examples/generate.sh +++ b/examples/generate.sh @@ -20,6 +20,10 @@ set -o nounset SOURCE_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" OUTPUT_DIR=${OUTPUT_DIR:-${SOURCE_DIR}/_out} +# import azinfo +source $SOURCE_DIR/aws-region-az.sh +set_region_az_info ${AWS_REGION} + # Binaries ENVSUBST=${ENVSUBST:-envsubst} command -v "${ENVSUBST}" >/dev/null 2>&1 || echo -v "Cannot find ${ENVSUBST} in path." diff --git a/hack/aws-region-az-fetch.sh b/hack/aws-region-az-fetch.sh new file mode 100755 index 0000000000..99a2c624c5 --- /dev/null +++ b/hack/aws-region-az-fetch.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +# Copyright 2019 The Kubernetes Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -o errexit +set -o nounset +set -o pipefail + +type aws || exit + +echo "fetching region list and AZ list" +for REGION in $(aws ec2 describe-regions --query "Regions[].{Name:RegionName}" --output text | paste -sd " " -); +do + AZLIST=$(aws ec2 describe-availability-zones --region $REGION --query "AvailabilityZones[].{Name:ZoneName}" --output text | paste -sd " " -) + echo "$REGION:$AZLIST," +done \ No newline at end of file From 96778a7f10360ca46400977082461e5a7bcd7832 Mon Sep 17 00:00:00 2001 From: guohaowang Date: Thu, 10 Oct 2019 16:18:23 +0800 Subject: [PATCH 2/2] feature: alb ingress controller support Signed-off-by: guohaowang --- examples/alb-ingress-controller-addons.yaml | 135 ++++++++++++++++++ examples/generate.sh | 15 ++ .../services/cloudformation/bootstrap.go | 106 ++++++++++++++ 3 files changed, 256 insertions(+) create mode 100644 examples/alb-ingress-controller-addons.yaml diff --git a/examples/alb-ingress-controller-addons.yaml b/examples/alb-ingress-controller-addons.yaml new file mode 100644 index 0000000000..29cfc43dd7 --- /dev/null +++ b/examples/alb-ingress-controller-addons.yaml @@ -0,0 +1,135 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + labels: + app.kubernetes.io/name: alb-ingress-controller + name: alb-ingress-controller +rules: + - apiGroups: + - "" + - extensions + resources: + - configmaps + - endpoints + - events + - ingresses + - ingresses/status + - services + verbs: + - create + - get + - list + - update + - watch + - patch + - apiGroups: + - "" + - extensions + resources: + - nodes + - pods + - secrets + - services + - namespaces + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + labels: + app.kubernetes.io/name: alb-ingress-controller + name: alb-ingress-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: alb-ingress-controller +subjects: + - kind: ServiceAccount + name: alb-ingress-controller + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + labels: + app.kubernetes.io/name: alb-ingress-controller + name: alb-ingress-controller + namespace: kube-system +--- +# Application Load Balancer (ALB) Ingress Controller Deployment Manifest. +# This manifest details sensible defaults for deploying an ALB Ingress Controller. +# GitHub: https://github.com/kubernetes-sigs/aws-alb-ingress-controller +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: alb-ingress-controller + name: alb-ingress-controller + # Namespace the ALB Ingress Controller should run in. Does not impact which + # namespaces it's able to resolve ingress resource for. For limiting ingress + # namespace scope, see --watch-namespace. + namespace: kube-system +spec: + selector: + matchLabels: + app.kubernetes.io/name: alb-ingress-controller + template: + metadata: + labels: + app.kubernetes.io/name: alb-ingress-controller + spec: + hostNetwork: true + tolerations: + - operator: Exists + containers: + - name: alb-ingress-controller + args: + # Limit the namespace where this ALB Ingress Controller deployment will + # resolve ingress resources. If left commented, all namespaces are used. + # - --watch-namespace=your-k8s-namespace + + # Setting the ingress-class flag below ensures that only ingress resources with the + # annotation kubernetes.io/ingress.class: "alb" are respected by the controller. You may + # choose any class you'd like for this controller to respect. + - --ingress-class=alb + + # REQUIRED + # Name of your cluster. Used when naming resources created + # by the ALB Ingress Controller, providing distinction between + # clusters. + - --cluster-name=${CLUSTER_NAME} + + # AWS VPC ID this ingress controller will use to create AWS resources. + # If unspecified, it will be discovered from ec2metadata. + # - --aws-vpc-id=vpc-xxxxxx + + # AWS region this ingress controller will operate in. + # If unspecified, it will be discovered from ec2metadata. + # List of regions: http://docs.aws.amazon.com/general/latest/gr/rande.html#vpc_region + # - --aws-region=us-west-1 + + # Enables logging on all outbound requests sent to the AWS API. + # If logging is desired, set to true. + # - --aws-api-debug + # Maximum number of times to retry the aws calls. + # defaults to 10. + # - --aws-max-retries=10 + # env: + # AWS key id for authenticating with the AWS API. + # This is only here for examples. It's recommended you instead use + # a project like kube2iam for granting access. + #- name: AWS_ACCESS_KEY_ID + # value: KEYVALUE + + # AWS key secret for authenticating with the AWS API. + # This is only here for examples. It's recommended you instead use + # a project like kube2iam for granting access. + #- name: AWS_SECRET_ACCESS_KEY + # value: SECRETVALUE + # Repository location of the ALB Ingress Controller. + image: docker.io/amazon/aws-alb-ingress-controller:v1.1.3 + serviceAccountName: alb-ingress-controller diff --git a/examples/generate.sh b/examples/generate.sh index 0e3cd726f7..2051485183 100755 --- a/examples/generate.sh +++ b/examples/generate.sh @@ -31,6 +31,9 @@ command -v "${ENVSUBST}" >/dev/null 2>&1 || echo -v "Cannot find ${ENVSUBST} in CLUSTERAWSADM=${CLUSTERAWSADM:-${SOURCE_DIR}/../bin/clusterawsadm} command -v "${CLUSTERAWSADM}" >/dev/null 2>&1 || echo -v "Cannot find ${CLUSTERAWSADM} in path, build it using 'make binaries' in this repository." +# NETWROK +export INGRESS="${INGRESS:-"disbale"}" + # Cluster. export CLUSTER_NAME="${CLUSTER_NAME:-test1}" export KUBERNETES_VERSION="${KUBERNETES_VERSION:-v1.15.3}" @@ -40,6 +43,9 @@ export CONTROL_PLANE_MACHINE_TYPE="${CONTROL_PLANE_MACHINE_TYPE:-t2.medium}" export NODE_MACHINE_TYPE="${CONTROL_PLANE_MACHINE_TYPE:-t2.medium}" export SSH_KEY_NAME="${SSH_KEY_NAME:-default}" +# ADDONS +ADDONS_GENERATED_FILE=${OUTPUT_DIR}/addons.yaml + # Outputs. COMPONENTS_CLUSTER_API_GENERATED_FILE=${SOURCE_DIR}/provider-components/provider-components-cluster-api.yaml COMPONENTS_KUBEADM_GENERATED_FILE=${SOURCE_DIR}/provider-components/provider-components-kubeadm.yaml @@ -87,6 +93,15 @@ fi mkdir -p "${OUTPUT_DIR}" +# Generate ADDONS. +cat "${SOURCE_DIR}/addons.yaml" | envsubst > "${ADDONS_GENERATED_FILE}" + +if [ ${INGRESS} == "ALB" ]; then + echo "Generated addons will include alb ingress controller" + cat "${SOURCE_DIR}/alb-ingress-controller-addons.yaml" | envsubst >> "${ADDONS_GENERATED_FILE}" +fi +echo "Generated ${ADDONS_GENERATED_FILE}" + # Generate AWS Credentials. AWS_B64ENCODED_CREDENTIALS="$(${CLUSTERAWSADM} alpha bootstrap encode-aws-credentials)" export AWS_B64ENCODED_CREDENTIALS diff --git a/pkg/cloud/services/cloudformation/bootstrap.go b/pkg/cloud/services/cloudformation/bootstrap.go index 85c362839b..eeca4eadc8 100644 --- a/pkg/cloud/services/cloudformation/bootstrap.go +++ b/pkg/cloud/services/cloudformation/bootstrap.go @@ -296,6 +296,112 @@ func cloudProviderControlPlaneAwsPolicy() *iam.PolicyDocument { "kms:DescribeKey", }, }, + { + Effect: iam.EffectAllow, + Resource: iam.Resources{"*"}, + Action: iam.Actions{ + "acm:DescribeCertificate", + "acm:ListCertificates", + "acm:GetCertificate", + }, + }, + { + Effect: iam.EffectAllow, + Resource: iam.Resources{"*"}, + Action: iam.Actions{ + "ec2:AuthorizeSecurityGroupIngress", + "ec2:CreateSecurityGroup", + "ec2:CreateTags", + "ec2:DeleteTags", + "ec2:DeleteSecurityGroup", + "ec2:DescribeAccountAttributes", + "ec2:DescribeAddresses", + "ec2:DescribeInstances", + "ec2:DescribeInstanceStatus", + "ec2:DescribeInternetGateways", + "ec2:DescribeNetworkInterfaces", + "ec2:DescribeSecurityGroups", + "ec2:DescribeSubnets", + "ec2:DescribeTags", + "ec2:DescribeVpcs", + "ec2:ModifyInstanceAttribute", + "ec2:ModifyNetworkInterfaceAttribute", + "ec2:RevokeSecurityGroupIngress", + }, + }, + { + Effect: iam.EffectAllow, + Resource: iam.Resources{"*"}, + Action: iam.Actions{ + "elasticloadbalancing:AddListenerCertificates", + "elasticloadbalancing:AddTags", + "elasticloadbalancing:CreateListener", + "elasticloadbalancing:CreateLoadBalancer", + "elasticloadbalancing:CreateRule", + "elasticloadbalancing:CreateTargetGroup", + "elasticloadbalancing:DeleteListener", + "elasticloadbalancing:DeleteLoadBalancer", + "elasticloadbalancing:DeleteRule", + "elasticloadbalancing:DeleteTargetGroup", + "elasticloadbalancing:DeregisterTargets", + "elasticloadbalancing:DescribeListenerCertificates", + "elasticloadbalancing:DescribeListeners", + "elasticloadbalancing:DescribeLoadBalancers", + "elasticloadbalancing:DescribeLoadBalancerAttributes", + "elasticloadbalancing:DescribeRules", + "elasticloadbalancing:DescribeSSLPolicies", + "elasticloadbalancing:DescribeTags", + "elasticloadbalancing:DescribeTargetGroups", + "elasticloadbalancing:DescribeTargetGroupAttributes", + "elasticloadbalancing:DescribeTargetHealth", + "elasticloadbalancing:ModifyListener", + "elasticloadbalancing:ModifyLoadBalancerAttributes", + "elasticloadbalancing:ModifyRule", + "elasticloadbalancing:ModifyTargetGroup", + "elasticloadbalancing:ModifyTargetGroupAttributes", + "elasticloadbalancing:RegisterTargets", + "elasticloadbalancing:RemoveListenerCertificates", + "elasticloadbalancing:RemoveTags", + "elasticloadbalancing:SetIpAddressType", + "elasticloadbalancing:SetSecurityGroups", + "elasticloadbalancing:SetSubnets", + "elasticloadbalancing:SetWebACL", + }, + }, + { + Effect: iam.EffectAllow, + Resource: iam.Resources{"*"}, + Action: iam.Actions{ + "iam:CreateServiceLinkedRole", + "iam:GetServerCertificate", + "iam:ListServerCertificates", + }, + }, + { + Effect: iam.EffectAllow, + Resource: iam.Resources{"*"}, + Action: iam.Actions{ + "waf-regional:GetWebACLForResource", + "waf-regional:GetWebACL", + "waf-regional:AssociateWebACL", + "waf-regional:DisassociateWebACL", + }, + }, + { + Effect: iam.EffectAllow, + Resource: iam.Resources{"*"}, + Action: iam.Actions{ + "tag:GetResources", + "tag:TagResources", + }, + }, + { + Effect: iam.EffectAllow, + Resource: iam.Resources{"*"}, + Action: iam.Actions{ + "waf:GetWebACL", + }, + }, }, } }