Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Updates to local setup scripts and introduce new local setup scripts for non-gardener use cases #852

Merged
merged 6 commits into from
Sep 22, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 42 additions & 9 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
# limitations under the License.

include .env
include hack/tools.mk

IMAGE_REPOSITORY := eu.gcr.io/gardener-project/gardener/machine-controller-manager
IMAGE_TAG := $(shell cat VERSION)
Expand All @@ -29,11 +30,11 @@ GOBIN=$(shell go env GOBIN)
endif

###########################################
# Rules When K8s cluster is Gardener Shoot#
# Setup targets for gardener shoot #
###########################################

.PHONY: setup
setup:
.PHONY: gardener-setup
gardener-setup:
@echo "enter project name"; \
read PROJECT; \
echo "enter seed name"; \
Expand All @@ -42,21 +43,53 @@ setup:
read SHOOT; \
echo "enter cluster provider(gcp|aws|azure|vsphere|openstack|alicloud|metal|equinix-metal)"; \
read PROVIDER; \
./hack/local_setup.sh --seed $$SEED --shoot $$SHOOT --project $$PROJECT --provider $$PROVIDER
./hack/gardener_local_setup.sh --seed $$SEED --shoot $$SHOOT --project $$PROJECT --provider $$PROVIDER

.PHONY: local-mcm-up
local-mcm-up: setup
.PHONY: gardener-local-mcm-up
gardener-local-mcm-up: gardener-setup
$(MAKE) start;

.PHONY: restore
restore:
.PHONY: gardener-restore
gardener-restore:
@echo "enter project name"; \
read PROJECT; \
echo "enter shoot name"; \
himanshu-kun marked this conversation as resolved.
Show resolved Hide resolved
read SHOOT; \
echo "enter cluster provider(gcp|aws|azure|vsphere|openstack|alicloud|metal|equinix-metal)"; \
read PROVIDER; \
./hack/local_restore.sh --shoot $$SHOOT --project $$PROJECT --provider $$PROVIDER
./hack/gardener_local_restore.sh --shoot $$SHOOT --project $$PROJECT --provider $$PROVIDER


###########################################
# Setup targets for non-gardener #
###########################################

.PHONY: non-gardener-setup
non-gardener-setup:
@echo "enter namespace"; \
read NAMESPACE; \
echo "enter control kubeconfig path"; \
read CONTROL_KUBECONFIG_PATH; \
echo "enter target kubeconfig path"; \
read TARGET_KUBECONFIG_PATH; \
echo "enter cluster provider(gcp|aws|azure|vsphere|openstack|alicloud|metal|equinix-metal)"; \
read PROVIDER; \
./hack/non_gardener_local_setup.sh --namespace $$NAMESPACE --control-kubeconfig-path $$CONTROL_KUBECONFIG_PATH --target-kubeconfig-path $$TARGET_KUBECONFIG_PATH --provider $$PROVIDER

.PHONY: non-gardener-local-mcm-up
non-gardener-local-mcm-up: non-gardener-setup
$(MAKE) start;

.PHONY: non-gardener-restore
non-gardener-restore:
@echo "enter namespace"; \
read NAMESPACE; \
echo "enter control kubeconfig path"; \
read CONTROL_KUBECONFIG_PATH; \
echo "enter cluster provider(gcp|aws|azure|vsphere|openstack|alicloud|metal|equinix-metal)"; \
read PROVIDER; \
@echo "enter project name"; \
./hack/non_gardener_local_restore.sh --namespace $$NAMESPACE --control-kubeconfig-path $$CONTROL_KUBECONFIG_PATH --provider $$PROVIDER

#########################################
# Rules for local development scenarios #
Expand Down
11 changes: 6 additions & 5 deletions docs/development/integration_tests.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@ Integration tests for `machine-controller-manager-provider-{provider-name}` can

1. Clone the repository `machine-controller-manager-provider-{provider-name}` on the local system.
1. Navigate to `machine-controller-manager-provider-{provider-name}` directory and create a `dev` sub-directory in it.
1. Copy the kubeconfig of the Control Cluster into `dev/control-kubeconfig.yaml` and update the `Makefile` variable `CONTROL_KUBECONFIG` to point to `dev/control-kubeconfig.yaml`.
1. Update the `Makefile` variable `CONTROL_NAMESPACE` to a valid namespace of the control cluster. This is the namespace that is used to deploy all resources and run tests.
1. (optional) Copy the kubeconfig of the Target Cluster into `dev/target-kubeconfig.yaml` and update the `Makefile` variable `TARGET_KUBECONFIG` to point to `dev/target-kubeconfig.yaml`.
1. If the tags on instances & associated resources on the provider are of `String` type (for example, GCP tags on its instances are of type `String` and not key-value pair) then add `TAGS_ARE_STRINGS := true` in the `Makefile` and export it.
1. Atleast, one of the two controllers' container images must be set in the `Makefile` variables `MCM_IMAGE` and `MC_IMAGE` for the controllers to run in the Control Cluster. These images will be used along with `kubernetes/deployment.yaml` to deploy/update controllers in the Control Cluster. If the intention is to run the controllers locally then unset the variables `MCM_IMAGE` and `MC_IMAGE` and set variable `MACHINE_CONTROLLER_MANAGER_DEPLOYMENT_NAME := machine-controller-manager` in the `Makefile`.
1. Create a `.env` file at the root of the `machine-controller-manager-provider-{provider-name}` project. This file serves as an environments file where all key-value pairs that are used in the `Makefile` are defined.
1. Copy the kubeconfig of the Control Cluster into `dev/control-kubeconfig.yaml` and add an entry in the `.env` file with `CONTROL_KUBECONFIG=dev/control-kubeconfig.yaml`.
1. Add `CONTROL_NAMESPACE=<namespace of the control cluster>` to the `.env` file. This is the namespace that is used to deploy all resources and run tests.
1. (optional) Copy the kubeconfig of the Target Cluster into `dev/target-kubeconfig.yaml` and add an entry in the `.env` file with `TARGET_KUBECONFIG=dev/target-kubeconfig.yaml`.
1. If the tags on instances & associated resources on the provider are of `String` type (for example, GCP tags on its instances are of type `String` and not key-value pair) then add `TAGS_ARE_STRINGS := true` in the `Makefile` and export it. For GCP this has already been hard coded in the `Makefile`.
1. If the intention is to run any controllers in the control cluster, then `.env` should have at least one of `MCM_IMAGE` and `MC_IMAGE` defined. These images will be used along with `kubernetes/deployment.yaml` to deploy/update controllers in the Control Cluster. If the intention is to run the controllers locally then remove `MCM_IMAGE` and `MC_IMAGE` key-value pairs defined in `.env` and set `MACHINE_CONTROLLER_MANAGER_DEPLOYMENT_NAME=machine-controller-manager` in the `.env` file.
1. In order to apply the CRDs when the Control Cluster is a Gardener Shoot or if none of the controller images are specified, `machine-controller-manager` repository will be cloned automatically. Incase, this repository already exists in local system, then create a softlink as below which helps to test changes in `machine-controller-manager` quickly.
```bash
ln -sf <path-for-machine-controller-manager-repo> dev/mcm
Expand Down
53 changes: 48 additions & 5 deletions docs/development/local_setup.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@ Create a Docker hub account at [Docker Hub](https://hub.docker.com/) if you don'
1. You have understood the [principles of Kubernetes](https://kubernetes.io/docs/concepts/), and its [components](https://kubernetes.io/docs/concepts/overview/components/), what their purpose is and how they interact with each other.
1. You have understood the [architecture of the Machine Controller Manager](../../README.md#design-of-machine-controller-manager)

The development of the Machine Controller Manager could happen by targetting any cluster. You basically need a Kubernetes cluster running on a set of machines. You just need the [Kubeconfig](https://kubernetes.io/docs/tasks/access-application-cluster/authenticate-across-clusters-kubeconfig/) file with the required access permissions attached to it.
The development of the Machine Controller Manager could happen by targeting any cluster. You basically need a Kubernetes cluster running on a set of machines. You just need the [Kubeconfig](https://kubernetes.io/docs/tasks/access-application-cluster/authenticate-across-clusters-kubeconfig/) file with the required access permissions attached to it.

### Installing the Machine Controller Manager locally
Clone the repository from GitHub.
Expand All @@ -66,10 +66,53 @@ $ kubectl apply -f kubernetes/crds.yaml

## Getting started

- Create a `dev` directory.
- Copy the kubeconfig of kubernetes cluster where you wish to deploy the machines into `dev/target-kubeconfig.yaml`.
- (optional) Copy the kubeconfig of kubernetes cluster from where you wish to manage the machines into `dev/control-kubeconfig.yaml`. If you do this, also update the `Makefile` variable CONTROL_KUBECONFIG to point to `dev/control-kubeconfig.yaml` and CONTROL_NAMESPACE to the namespace in which your controller watches over.
- There is a rule dev in the `Makefile` which will automatically start the Machine Controller Manager with development settings:
**Setup and Restore with Gardener**

_Setup_

In gardener access to static kubeconfig files is no longer supported due to security reasons. One needs to generate short-lived (max TTL = 1 day) admin kube configs for target and control clusters.
A convenience script/Makefile target has been provided to do the required initial setup which includes:
* Creating a temporary directory where target and control kubeconfigs will be stored.
* Create a request to generate the short lived admin kubeconfigs. These are downloaded and stored in the temporary folder created above.
* In gardener clusters `DWD (Dependency Watchdog)` runs as an additional component which can interfere when MCM/CA is scaled down. To prevent that an annotation `dependency-watchdog.gardener.cloud/ignore-scaling` is added to `machine-controller-manager` deployment which prevents `DWD` from scaling up the deployment replicas.
* Scales down `machine-controller-manager` deployment in the control cluster to 0 replica.
* Creates the required `.env` file and populates required environment variables which are then used by the `Makefile` in both `machine-controller-manager` and in `machine-controller-manager-provider-<provider-name>` projects.
* Copies the generated and downloaded kubeconfig files for the target and control clusters to `machine-controller-manager-provider-<provider-name>` project as well.

To do the above you can either invoke `make gardener-setup` or you can directly invoke the script `./hack/gardener_local_setup.sh`. If you invoke the script with `-h or --help` option then it will give you all CLI options that one can pass.

_Restore_

Once the testing is over you can invoke a convenience script/Makefile target which does the following:
* Removes all generated admin kubeconfig files from both `machine-controller-manager` and in `machine-controller-manager-provider-<provider-name>` projects.
* Removes the `.env` file that was generated as part of the setup from both `machine-controller-manager` and in `machine-controller-manager-provider-<provider-name>` projects.
* Scales up `machine-controller-manager` deployment in the control cluster back to 1 replica.
* Removes the annotation `dependency-watchdog.gardener.cloud/ignore-scaling` that was added to prevent `DWD` to scale up MCM.

To do the above you can either invoke `make gardener-restore` or you can directly invoke the script `./hack/gardener_local_restore.sh`. If you invoke the script with `-h or --help` option then it will give you all CLI options that one can pass.

**Setup and Restore without Gardener**

_Setup_

If you are not running MCM components in a gardener cluster, then it is assumed that there is not going to be any `DWD (Dependency Watchdog)` component.
A convenience script/Makefile target has been provided to the required initial setup which includes:
* Copies the provided control and target kubeconfig files to `machine-controller-manager-provider-<provider-name>` project.
* Scales down `machine-controller-manager` deployment in the control cluster to 0 replica.
* Creates the required `.env` file and populates required environment variables which are then used by the `Makefile` in both `machine-controller-manager` and in `machine-controller-manager-provider-<provider-name>` projects.

To do the above you can either invoke `make non-gardener-setup` or you can directly invoke the script `./hack/non_gardener_local_setup.sh`. If you invoke the script with `-h or --help` option then it will give you all CLI options that one can pass.

_Restore_

Once the testing is over you can invoke a convenience script/Makefile target which does the following:
* Removes all provided kubeconfig files from both `machine-controller-manager` and in `machine-controller-manager-provider-<provider-name>` projects.
* Removes the `.env` file that was generated as part of the setup from both `machine-controller-manager` and in `machine-controller-manager-provider-<provider-name>` projects.
* Scales up `machine-controller-manager` deployment in the control cluster back to 1 replica.

To do the above you can either invoke `make non-gardener-restore` or you can directly invoke the script `./hack/non_gardener_local_restore.sh`. If you invoke the script with `-h or --help` option then it will give you all CLI options that one can pass.

Once the setup is done then you can start the `machine-controller-manager` as a local process using the following `Makefile` target:

```bash
$ make start
Expand Down
4 changes: 2 additions & 2 deletions hack/local_restore.sh → hack/gardener_local_restore.sh
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ set -o pipefail

##############################################################################################################
# Script restores MCM deployment and removes the annotation dependency-watchdog.gardener.cloud/ignore-scaling
# This script should be called once you are done using local_setup.sh
# This script should be called once you are done using gardener_local_setup.sh
##############################################################################################################

declare SHOOT PROJECT PROVIDER
Expand Down Expand Up @@ -118,7 +118,7 @@ function delete_generated_configs() {
echo "Clearing .env files..."
sed -r -i '/^#/!d' "${PROJECT_DIR}"/.env
sed -r -i '/^#/!d' "${PROVIDER_MCM_PROJECT_DIR}"/.env
echo "Removing generated admin kube config json..."
echo "Removing generated admin kube config json if any..."
rm -f "${SCRIPT_DIR}"/admin-kube-config-request.json
}

Expand Down
28 changes: 19 additions & 9 deletions hack/local_setup.sh → hack/gardener_local_setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ function create_usage() {
-l | --landscape <landscape-name> (Optional) Name of the landscape. Defaults to dev
-k | --kubeconfig-path <relative-kubeconfig-path> (Optional) Relative path to the <PROJECT-DIR> where kubeconfigs will be downloaded. Path should not start with '/'. Default: <PROJECT-DIR>/dev/kube-configs
-m | --mcm-provider-project-path <absolute-mcm-provider-project-dir> (Optional) MCM Provider project directory. If not provided then it assumes that both mcm and mcm-provider projects are under the same parent directory
-e | --kubeconfig-expiry-seconds <expiry-duration-in-seconds> (Optional) Common expiry durations in seconds for control and target KubeConfigs. Default: 3600 seconds
-e | --kubeconfig-expiry-seconds <expiry-duration-in-seconds> (Optional) Common expiry durations in seconds for control and target KubeConfigs. Default: 3600 seconds. Max accepted value is 86400 seconds.
")
echo "${usage}"
}
Expand Down Expand Up @@ -118,6 +118,10 @@ function validate_args() {
echo -e "Infrastructure provider name has not been passed. Please provide infrastructure provider name either by specifying --provider or -i argument"
exit 1
fi
if [[ -n "${KUBECONFIG_EXPIRY_SECONDS}" ]] && [[ "${KUBECONFIG_EXPIRY_SECONDS}" -gt 86400 ]]; then
echo -e "KubeConfig expiration seconds cannot be greater than 86400"
exit 1
fi
}

function initialize() {
Expand All @@ -126,7 +130,10 @@ function initialize() {
fi
echo "Creating directory ${RELATIVE_KUBECONFIG_PATH} if it does not exist..."
mkdir -p "${RELATIVE_KUBECONFIG_PATH}"
ABSOLUTE_KUBE_CONFIG_PATH="$(cd "$(dirname "${RELATIVE_KUBECONFIG_PATH}")"; pwd)/$(basename "${RELATIVE_KUBECONFIG_PATH}")"
ABSOLUTE_KUBE_CONFIG_PATH="$(
cd "$(dirname "${RELATIVE_KUBECONFIG_PATH}")"
pwd
)/$(basename "${RELATIVE_KUBECONFIG_PATH}")"
ABSOLUTE_PROVIDER_KUBECONFIG_PATH="${PROVIDER_MCM_PROJECT_DIR}/dev/kube-configs"
echo "Creating directory ${ABSOLUTE_PROVIDER_KUBECONFIG_PATH} if it does not exist..."
mkdir -p "${ABSOLUTE_PROVIDER_KUBECONFIG_PATH}"
Expand Down Expand Up @@ -156,6 +163,9 @@ function download_kubeconfigs() {
--raw "/apis/core.gardener.cloud/v1beta1/namespaces/${project_namespace}/shoots/${SHOOT}/adminkubeconfig" |
jq -r '.status.kubeconfig' |
base64 -d >"${ABSOLUTE_KUBE_CONFIG_PATH}/kubeconfig_target.yaml"

echo "Removing generated admin kube config json..."
rm -f "${SCRIPT_DIR}"/admin-kube-config-request.json
}

function create_kubeconfig_request_yaml() {
Expand Down Expand Up @@ -192,15 +202,15 @@ function scale_down_mcm() {
function set_makefile_env() {
if [[ $# -ne 2 ]]; then
echo -e "${FUNCNAME[0]} expects two arguments - project-directory and target-kube-config-path"
fi
local target_project_dir target_kube_config_path
target_project_dir="$1"
target_kube_config_path="$2"
fi
local target_project_dir target_kube_config_path
target_project_dir="$1"
target_kube_config_path="$2"
{
printf "\n%s" "CONTROL_NAMESPACE=shoot--${PROJECT}--${SHOOT}"
printf "\n%s" "CONTROL_KUBECONFIG=${target_kube_config_path}/kubeconfig_control.yaml" >> "${target_project_dir}/.env"
printf "\n%s" "TARGET_KUBECONFIG=${target_kube_config_path}/kubeconfig_target.yaml" >> "${target_project_dir}/.env"
} >> "${target_project_dir}/.env"
printf "\n%s" "CONTROL_KUBECONFIG=${target_kube_config_path}/kubeconfig_control.yaml" >>"${target_project_dir}/.env"
printf "\n%s" "TARGET_KUBECONFIG=${target_kube_config_path}/kubeconfig_target.yaml" >>"${target_project_dir}/.env"
} >>"${target_project_dir}/.env"
}

function main() {
Expand Down
Loading