Prerequisite

pull git repos

git clone https://github.com/itohdak/kubernetes-experiment.git # clone this repo
cd kubernetes-experiment

git clone -b dev https://github.com/itohdak/microservices-demo.git
git clone -b dev https://github.com/itohdak/locust-experiments.git
# git clone https://github.com/prometheus-operator/kube-prometheus.git

set env

PROJECT_ID=[your-project-id]
ZONE=asia-northeast1
CLUSTER_NAME=[your-cluster-name]

example

PROJECT_ID=charming-scarab-316315
ZONE=asia-northeast1
CLUSTER_NAME=onlineboutique

Tips (GKE)

before starting (enable access from local terminal)

gcloud init
gcloud services enable container.googleapis.com

ref

• gcloud install

create GKE cluster

gcloud container clusters create ${CLUSTER_NAME} \
    --project=${PROJECT_ID} --zone=${ZONE} \
    --node-locations ${ZONE}-a,${ZONE}-b \
    --machine-type=e2-standard-4 --num-nodes=2

scale in / out

gcloud container clusters resize $CLUSTER_NAME --size 0 --zone $ZONE
gcloud container clusters resize $CLUSTER_NAME --size 2 --zone $ZONE

accessing gcloud from local using kubectl

gcloud container clusters get-credentials ${CLUSTER_NAME} --zone ${ZONE}

ref

• enable kubectl

deploy istio

istioctl install --set profile=demo -y
kubectl label namespace default istio-injection=enabled

if istioctl is not installed

curl -L https://istio.io/downloadIstio | sh -
# install specific version
# curl -L https://istio.io/downloadIstio | ISTIO_VERSION=1.10.1 TARGET_ARCH=x86_64 sh -

cd istio-1.10.1
echo export PATH=$PWD/bin:'$PATH' >> ~/.bashrc
exec bash # or exec $SHELL -l for mac

then

which istioctl

will return your istio path.

taint node for locust

LOCUST_NODE=`kubectl get nodes -ojsonpath='{.items[].metadata.name}'`
kubectl taint node $LOCUST_NODE run=locust:NoSchedule
kubectl label nodes $LOCUST_NODE app=locust

deploy

kubectl apply -f ./microservices-demo/release/kubernetes-manifests.yaml
kubectl apply -f ./microservices-demo/release/istio-manifests.yaml

get access path

INGRESS_HOST="$(kubectl -n istio-system get service istio-ingressgateway \
   -o jsonpath='{.status.loadBalancer.ingress[0].ip}')"
echo "$INGRESS_HOST"

You can now access the sample application on http://$INGRESS_HOST.

Prometheus

along with istio (most easy for me)

cd $ISTIO_HOME/samples/addons/
kubectl apply -f prometheus.yaml
kubectl apply -f grafana.yaml

access prometheus/grafana via port-forwarding

kubectl -n istio-system port-forward svc/prometheus 9090:9090
kubectl -n istio-system port-forward svc/grafana 3000:3000

without helm

git clone https://github.com/prometheus-operator/kube-prometheus.git
cd kube-prometheus
kubectl create -f manifests/setup
until kubectl get servicemonitors --all-namespaces ; do date; sleep 1; echo ""; done
kubectl create -f manifests/

access prometheus/grafana via port-forwarding

kubectl --namespace monitoring port-forward svc/prometheus-k8s 9090
kubectl --namespace monitoring port-forward svc/grafana 3000

key	value
user	admin
password	admin

with helm

kubectl create ns monitoring
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
helm repo add stable https://charts.helm.sh/stable
helm repo update
helm -n monitoring install prometheus-operator prometheus-community/kube-prometheus-stack

install prometheus-stack via heml helm install error

access grafana via port-forwarding

kubectl -n monitoring port-forward svc/prometheus-operator-kube-p-prometheus 9090:9090
kubectl -n monitoring port-forward svc/prometheus-operator-grafana 3000:80

key	value
user	admin
password	prom-operator

locust

kubectl apply -f ./locust-experiments/kubernetes
# deploys
# - locust-cm.yaml
# - scripts-cm.yaml
# - master-deployment.yaml
# - service.yaml
# - slave-deployment.yaml

# curl "http://localhost:8089/swarm" -X POST -H "Content-Type: application/x-www-form-urlencoded" --data "locust_count=200&hatch_rate=1"
curl "http://localhost:8089/swarm" -X POST -H "Content-Type: application/x-www-form-urlencoded; charset=UTF-8" --data "user_count=200&spawn_rate=1"

load test

kubectl port-forward svc/locust-master 8089:8089 

Installation

Flagger

ref

$ helm repo add flagger https://flagger.app
"flagger" has been added to your repositories

$ kubectl apply -f https://raw.githubusercontent.com/fluxcd/flagger/main/artifacts/flagger/crd.yaml
customresourcedefinition.apiextensions.k8s.io/canaries.flagger.app created
customresourcedefinition.apiextensions.k8s.io/metrictemplates.flagger.app created
customresourcedefinition.apiextensions.k8s.io/alertproviders.flagger.app created

$ helm upgrade -i flagger flagger/flagger \
 --namespace=istio-system \
 --set crd.create=false \
 --set meshProvider=istio \
 --set metricsServer=http://prometheus:9090
Release "flagger" does not exist. Installing it now.
NAME: flagger
LAST DEPLOYED: Sat Jul 17 15:32:11 2021
NAMESPACE: istio-system
STATUS: deployed
REVISION: 1
TEST SUITE: None
NOTES:
Flagger installed

Metrics check in Flagger

ref

Golang

ref

$ wget https://dl.google.com/go/go1.16.6.linux-amd64.tar.gz
--2021-07-17 15:51:45--  https://dl.google.com/go/go1.16.6.linux-amd64.tar.gz
Resolving dl.google.com (dl.google.com)... 2404:6800:4004:81d::200e, 172.217.175.46
Connecting to dl.google.com (dl.google.com)|2404:6800:4004:81d::200e|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 129049323 (123M) [application/x-gzip]
Saving to: ‘go1.16.6.linux-amd64.tar.gz’

go1.16.6.linux-amd64.tar.gz               100%[===================================================================================>] 123.07M  70.8MB/s    in 1.7s    

2021-07-17 15:51:47 (70.8 MB/s) - ‘go1.16.6.linux-amd64.tar.gz’ saved [129049323/129049323]

$ sudo tar -C /usr/local -xzf go1.16.6.linux-amd64.tar.gz

$ echo export PATH=/usr/local/go/bin:'$PATH' >> ~/.bashrc

get Prometheus exported metrics with golang

$ go get github.com/prometheus/client_golang/api
$ go get github.com/prometheus/client_golang/api/prometheus/v1

Helm

curl -fsSL -o get_helm.sh https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3
chmod 700 get_helm.sh
./get_helm.sh