From 59a2ad0d9e071d450d1ee64c741c282b9a29a9e8 Mon Sep 17 00:00:00 2001 From: rpudlowski93 <33381523+rpudlowski93@users.noreply.github.com> Date: Mon, 4 Jan 2021 15:53:25 +0100 Subject: [PATCH] Ability do deploy Istio (#1925) * Istio * Dockerfile updated for devcontainer * Documents updated * Istio namespaces creating added * Documentation improved * Hub address added to profile configuration file * Istio added to local registry * HowTo updated and task with deploying istio improved * HowTo doc improved * Correct verb form * Small changes in HowTo doc and DockerFile Co-authored-by: to-bar <46519524+to-bar@users.noreply.github.com> --- CHANGELOG-0.9.md | 1 + Dockerfile | 6 ++ core/src/epicli/.devcontainer/Dockerfile | 6 ++ .../istio/deploy-istio-operator.yml | 25 +++++ .../tasks/applications/istio/main.yml | 9 ++ .../templates/istio/00-namespace.yml.j2 | 14 +++ .../templates/istio/01-profile.yml.j2 | 14 +++ .../centos-7/requirements.txt | 4 + .../redhat-7/requirements.txt | 4 + .../ubuntu-18.04/requirements.txt | 4 + .../defaults/configuration/applications.yml | 14 +++ .../defaults/configuration/image-registry.yml | 6 ++ docs/home/COMPONENTS.md | 2 + docs/home/howto/ISTIO.md | 96 +++++++++++++++++++ 14 files changed, 205 insertions(+) create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/applications/tasks/applications/istio/deploy-istio-operator.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/applications/tasks/applications/istio/main.yml create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/applications/templates/istio/00-namespace.yml.j2 create mode 100644 core/src/epicli/data/common/ansible/playbooks/roles/applications/templates/istio/01-profile.yml.j2 create mode 100644 docs/home/howto/ISTIO.md diff --git a/CHANGELOG-0.9.md b/CHANGELOG-0.9.md index 3e97b01c01..06fb79c3b8 100644 --- a/CHANGELOG-0.9.md +++ b/CHANGELOG-0.9.md @@ -5,6 +5,7 @@ ### Added - [#921](https://github.com/epiphany-platform/epiphany/issues/921) - Implement log rotation for PgBouncer +- [#1911](https://github.com/epiphany-platform/epiphany/issues/1911) - Ability to deploy Istio ### Fixed diff --git a/Dockerfile b/Dockerfile index 28ca45eb0f..eb5816b5cc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ FROM python:3.7-slim ARG HELM_VERSION=3.3.1 ARG KUBECTL_VERSION=1.18.8 +ARG ISTIOCTL_VERSION=1.8.1 ARG USERNAME=epiuser ARG USER_UID=1000 @@ -36,6 +37,11 @@ RUN apt-get update \ && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl \ && kubectl version --client \ + && echo "Installing istioctl binary ..." \ + && curl -fsSLO https://github.com/istio/istio/releases/download/${ISTIOCTL_VERSION}/istioctl-${ISTIOCTL_VERSION}-linux-amd64.tar.gz \ + && tar -xzof istioctl-${ISTIOCTL_VERSION}-linux-amd64.tar.gz -C /usr/local/bin istioctl \ + && rm istioctl-${ISTIOCTL_VERSION}-linux-amd64.tar.gz \ + && chmod +x /usr/local/bin/istioctl \ \ && setcap 'cap_net_bind_service=+ep' /usr/bin/ssh \ \ diff --git a/core/src/epicli/.devcontainer/Dockerfile b/core/src/epicli/.devcontainer/Dockerfile index 300205ce68..0d7e77dfdb 100644 --- a/core/src/epicli/.devcontainer/Dockerfile +++ b/core/src/epicli/.devcontainer/Dockerfile @@ -2,6 +2,7 @@ FROM python:3.7-slim ARG HELM_VERSION=3.3.1 ARG KUBECTL_VERSION=1.18.8 +ARG ISTIOCTL_VERSION=1.8.1 ARG USERNAME=vscode ARG USER_UID=1000 @@ -35,6 +36,11 @@ RUN chmod +x /config-pre.sh \ && chmod +x ./kubectl \ && mv ./kubectl /usr/local/bin/kubectl \ && kubectl version --client \ + && echo "Installing istioctl binary ..." \ + && curl -fsSLO https://github.com/istio/istio/releases/download/${ISTIOCTL_VERSION}/istioctl-${ISTIOCTL_VERSION}-linux-amd64.tar.gz \ + && tar -xzof istioctl-${ISTIOCTL_VERSION}-linux-amd64.tar.gz -C /usr/local/bin istioctl \ + && rm istioctl-${ISTIOCTL_VERSION}-linux-amd64.tar.gz \ + && chmod +x /usr/local/bin/istioctl \ \ && setcap 'cap_net_bind_service=+ep' /usr/bin/ssh \ \ diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/applications/tasks/applications/istio/deploy-istio-operator.yml b/core/src/epicli/data/common/ansible/playbooks/roles/applications/tasks/applications/istio/deploy-istio-operator.yml new file mode 100644 index 0000000000..05d513a5a5 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/applications/tasks/applications/istio/deploy-istio-operator.yml @@ -0,0 +1,25 @@ +--- + +- name: Decide if internal docker registry will be used + set_fact: + use_epiphany_image_registry: >- + {{ (not _k8s_as_cloud_service) and _use_local_image_registry }} + vars: + # Be extra paranoid + _k8s_as_cloud_service: >- + {{ k8s_as_cloud_service | bool }} + # Enable by default + _use_local_image_registry: >- + {{ (data.use_local_image_registry is undefined) or (data.use_local_image_registry | bool) }} + +- name: Init istio operator + command: istioctl operator init {{ _init_args | join(' ') }} + vars: + _common_args: + - --istioNamespace={{ data.namespaces.istio }} + - --operatorNamespace={{ data.namespaces.operator }} + - --watchedNamespaces={{ data.namespaces.watched | join(',') }} + _hub_arg: --hub={{ image_registry_address }}/istio + _init_args: >- + {{ _common_args + [ _hub_arg ] if (use_epiphany_image_registry) else + _common_args }} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/applications/tasks/applications/istio/main.yml b/core/src/epicli/data/common/ansible/playbooks/roles/applications/tasks/applications/istio/main.yml new file mode 100644 index 0000000000..14ae5f6a0a --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/applications/tasks/applications/istio/main.yml @@ -0,0 +1,9 @@ +--- + +- name: Deploy Istio operator + include_tasks: deploy-istio-operator.yml + +- name: Deploy Istio + include_tasks: deploy-k8s-app.yml + vars: + app_dir_name: "{{ data.name }}" diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/applications/templates/istio/00-namespace.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/applications/templates/istio/00-namespace.yml.j2 new file mode 100644 index 0000000000..fe8220374a --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/applications/templates/istio/00-namespace.yml.j2 @@ -0,0 +1,14 @@ +{% for namespace in data.namespaces.watched %} +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ namespace }} +{% endfor %} +{% if data.namespaces.istio not in data.namespaces.watched %} +--- +apiVersion: v1 +kind: Namespace +metadata: + name: {{ data.namespaces.istio }} +{% endif %} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/applications/templates/istio/01-profile.yml.j2 b/core/src/epicli/data/common/ansible/playbooks/roles/applications/templates/istio/01-profile.yml.j2 new file mode 100644 index 0000000000..76e5728253 --- /dev/null +++ b/core/src/epicli/data/common/ansible/playbooks/roles/applications/templates/istio/01-profile.yml.j2 @@ -0,0 +1,14 @@ +--- +apiVersion: install.istio.io/v1alpha1 +kind: IstioOperator +metadata: + namespace: {{ data.namespaces.watched[0] }} + name: {{ data.istio_spec.name }} +spec: + profile: {{ data.istio_spec.profile }} +{% if use_epiphany_image_registry %} + hub: {{ image_registry_address }}/istio +{% endif %} + values: + global: + istioNamespace: {{ data.namespaces.istio }} diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt index bb268fe371..fc2eb97892 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/centos-7/requirements.txt @@ -173,6 +173,10 @@ vault:1.3.2 apacheignite/ignite:2.5.0 bitnami/pgpool:4.1.1-debian-10-r29 brainsam/pgbouncer:1.12 +# istio +istio/pilot:1.8.1 +istio/proxyv2:1.8.1 +istio/operator:1.8.1 # TODO remove? jboss/keycloak:4.8.3.Final jboss/keycloak:9.0.0 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt index 7c868919a3..8478531ebf 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/redhat-7/requirements.txt @@ -170,6 +170,10 @@ vault:1.3.2 apacheignite/ignite:2.5.0 bitnami/pgpool:4.1.1-debian-10-r29 brainsam/pgbouncer:1.12 +# istio +istio/pilot:1.8.1 +istio/proxyv2:1.8.1 +istio/operator:1.8.1 # TODO remove? jboss/keycloak:4.8.3.Final jboss/keycloak:9.0.0 diff --git a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt index 2ae0ca216e..e4a3d75f72 100644 --- a/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt +++ b/core/src/epicli/data/common/ansible/playbooks/roles/repository/files/download-requirements/ubuntu-18.04/requirements.txt @@ -219,6 +219,10 @@ vault:1.3.2 apacheignite/ignite:2.5.0 bitnami/pgpool:4.1.1-debian-10-r29 brainsam/pgbouncer:1.12 +# istio +istio/pilot:1.8.1 +istio/proxyv2:1.8.1 +istio/operator:1.8.1 # TODO remove? jboss/keycloak:4.8.3.Final jboss/keycloak:9.0.0 diff --git a/core/src/epicli/data/common/defaults/configuration/applications.yml b/core/src/epicli/data/common/defaults/configuration/applications.yml index 10b15e9da4..fafd507fd5 100644 --- a/core/src/epicli/data/common/defaults/configuration/applications.yml +++ b/core/src/epicli/data/common/defaults/configuration/applications.yml @@ -169,3 +169,17 @@ specification: DEFAULT_POOL_SIZE: 25 RESERVE_POOL_SIZE: 25 POOL_MODE: transaction + +## --- istio --- + + - name: istio + enabled: false + use_local_image_registry: true + namespaces: + operator: istio-operator # namespace where operator will be deployed + watched: # list of namespaces which operator will watch + - istio-system + istio: istio-system # namespace where istio control plane will be deployed + istio_spec: + profile: default # Check all possibilites https://istio.io/latest/docs/setup/additional-setup/config-profiles/ + name: istiocontrolplane diff --git a/core/src/epicli/data/common/defaults/configuration/image-registry.yml b/core/src/epicli/data/common/defaults/configuration/image-registry.yml index e49edacce2..d97d038d68 100644 --- a/core/src/epicli/data/common/defaults/configuration/image-registry.yml +++ b/core/src/epicli/data/common/defaults/configuration/image-registry.yml @@ -22,6 +22,12 @@ specification: file_name: vault-1.3.2.tar - name: "hashicorp/vault-k8s:0.2.0" file_name: vault-k8s-0.2.0.tar + - name: "istio/proxyv2:1.8.1" + file_name: proxyv2-1.8.1.tar + - name: "istio/pilot:1.8.1" + file_name: pilot-1.8.1.tar + - name: "istio/operator:1.8.1" + file_name: operator-1.8.1.tar # postgres - name: bitnami/pgpool:4.1.1-debian-10-r29 file_name: pgpool-4.1.1-debian-10-r29.tar diff --git a/docs/home/COMPONENTS.md b/docs/home/COMPONENTS.md index f3258759f9..b95a25790b 100644 --- a/docs/home/COMPONENTS.md +++ b/docs/home/COMPONENTS.md @@ -41,6 +41,8 @@ Note that versions are default versions and can be changed in certain cases thro | Apache2 | 2.4.29 | https://httpd.apache.org/ | [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0) | | Hashicorp Vault | 1.4.0 | https://github.com/hashicorp/vault | [Mozilla Public License 2.0](https://github.com/hashicorp/vault/blob/master/LICENSE) | | Hashicorp Vault Helm Chart | 0.4.0 | https://github.com/hashicorp/vault-helm | [Mozilla Public License 2.0](https://github.com/hashicorp/vault-helm/blob/master/LICENSE.md) | +| Istio | 1.8.1 | https://github.com/istio/istio | [Apache License 2.0](https://github.com/istio/istio/blob/master/LICENSE) | + ## Epicli binary dependencies diff --git a/docs/home/howto/ISTIO.md b/docs/home/howto/ISTIO.md new file mode 100644 index 0000000000..ab1917e43d --- /dev/null +++ b/docs/home/howto/ISTIO.md @@ -0,0 +1,96 @@ +## Istio + +Open source platform which allows you to run service mesh for distributed microservice architecture. It allows to connect, manage and run secure connections between microservices and brings lots of features such as load balancing, monitoring and service-to-service authentication without any changes in service code. Read more about Istio [here](https://istio.io/latest/docs/concepts/what-is-istio/). + +### Installing Istio + +Istio in Epiphany is provided as K8s application. By default, it is not installed. To deploy it you need to add "configuration/applications" document to your configuration yaml file, similar to the example below (`enabled` flag must be set as `true`): + +Istio is installed using Istio Operator. Operator is a software extension to the Kubernetes API which has a deep knowledge how Istio deployments should look like and how to react if any problem appears. It is also very easy to make upgrades and automate tasks that would normally be executed by user/admin. + +```yaml +--- +kind: configuration/applications +version: 0.8.0 +title: "Kubernetes Applications Config" +provider: aws +name: default +specification: + applications: + ... + +## --- istio --- + + - name: istio + enabled: true + use_local_image_registry: true + namespaces: + operator: istio-operator # namespace where operator will be deployed + watched: # list of namespaces which operator will watch + - istio-system + istio: istio-system # namespace where Istio control plane will be deployed + istio_spec: + profile: default # Check all possibilites https://istio.io/latest/docs/setup/additional-setup/config-profiles/ + name: istiocontrolplane + +``` + +Using this configuration file, controller will detect Istio Operator resource in first of watched namespaces and will install Istio components corresponding to the specified profile (default). Using the default profile, Istio control plane and Istio ingress gateway will be deployed in istio-system namespace. + +### How to set up service mesh for an application + +The default Istio installation uses automcatic sidecar injection. You need to label the namespace where application will be hosted: + +```bash +kubectl label namespace default istio-injection=enabled +``` + +Once the proper namespaces are labeled and Istio is deployed, you can deploy your applications or restart existing ones. + +You may need to make an application accessible from outside of your Kubernetes cluster. An Istio Gateway which was deployed using default profile is used for this purpose. Define the ingress gateway deploying gateway and virtual service specification. The gateway specification describes the L4-L6 properties of a load balancer and the virtual service specification describes the L7 properties of a load balancer. + +Example of the gateway and virtual service specification (You have to adapt the entire specification to the application): + +[Gateway](https://istio.io/latest/docs/reference/config/networking/gateway/): + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: Gateway +metadata: + name: httpbin-gateway +spec: + selector: + istio: ingressgateway # use Istio default gateway implementation + servers: + - port: + number: 80 + name: http + protocol: HTTP + hosts: + - "httpbin.example.com" +``` + +[Virtual Service](https://istio.io/latest/docs/reference/config/networking/virtual-service/): + +```yaml +apiVersion: networking.istio.io/v1alpha3 +kind: VirtualService +metadata: + name: httpbin +spec: + hosts: + - "httpbin.example.com" + gateways: + - httpbin-gateway + http: + - match: + - uri: + prefix: /status + - uri: + prefix: /delay + route: + - destination: + port: + number: 8000 + host: httpbin +```