From f6dcf09bdceb5668010058c637e0d206111ee6ae Mon Sep 17 00:00:00 2001 From: "yingqi.ge" Date: Tue, 25 May 2021 14:30:42 +0800 Subject: [PATCH] Add node resource manager features --- CHANGELOG.md | 11 + CODE_OF_CONDUCT.md | 5 + Makefile | 20 + PROJECT | 3 + README.md | 49 +++ README.zh.md | 53 +++ build/build.sh | 118 ++++++ build/entrypoint.sh | 30 ++ deploy/configmap.yaml | 61 +++ deploy/nrm.yaml | 92 +++++ docs/configmap.md | 148 +++++++ docs/configmap.zh.md | 152 +++++++ docs/developer-guide.md | 15 + docs/images/node-resource-manager.png | Bin 0 -> 55417 bytes docs/roadmap.md | 3 + go.mod | 23 ++ go.sum | 388 ++++++++++++++++++ main.go | 123 ++++++ pkg/config/cloud.go | 104 +++++ pkg/config/global.go | 71 ++++ pkg/manager/manager.go | 113 +++++ pkg/manager/memory/memory.go | 114 ++++++ pkg/manager/memory/memory_test.go | 127 ++++++ pkg/manager/memory/type.go | 32 ++ pkg/manager/quotapath/quotapath.go | 175 ++++++++ pkg/manager/quotapath/quotapath_test.go | 151 +++++++ pkg/manager/quotapath/type.go | 35 ++ pkg/manager/volumegroup/recorder.go | 17 + pkg/manager/volumegroup/type.go | 32 ++ pkg/manager/volumegroup/utils.go | 270 ++++++++++++ pkg/manager/volumegroup/volumegroup.go | 432 ++++++++++++++++++++ pkg/manager/volumegroup/volumegroup_test.go | 200 +++++++++ pkg/model/type.go | 417 +++++++++++++++++++ pkg/signals/signal_posix.go | 25 ++ pkg/signals/signals.go | 82 ++++ pkg/utils/lvm.go | 297 ++++++++++++++ pkg/utils/lvm_mock.go | 214 ++++++++++ pkg/utils/mounter.go | 263 ++++++++++++ pkg/utils/mounter_mock.go | 107 +++++ pkg/utils/pmem.go | 173 ++++++++ pkg/utils/pmem_mock.go | 135 ++++++ pkg/utils/utils.go | 176 ++++++++ 42 files changed, 5056 insertions(+) create mode 100644 CHANGELOG.md create mode 100644 CODE_OF_CONDUCT.md create mode 100644 Makefile create mode 100644 PROJECT create mode 100644 README.md create mode 100644 README.zh.md create mode 100755 build/build.sh create mode 100644 build/entrypoint.sh create mode 100644 deploy/configmap.yaml create mode 100644 deploy/nrm.yaml create mode 100644 docs/configmap.md create mode 100644 docs/configmap.zh.md create mode 100644 docs/developer-guide.md create mode 100644 docs/images/node-resource-manager.png create mode 100644 docs/roadmap.md create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 pkg/config/cloud.go create mode 100644 pkg/config/global.go create mode 100644 pkg/manager/manager.go create mode 100644 pkg/manager/memory/memory.go create mode 100644 pkg/manager/memory/memory_test.go create mode 100644 pkg/manager/memory/type.go create mode 100644 pkg/manager/quotapath/quotapath.go create mode 100644 pkg/manager/quotapath/quotapath_test.go create mode 100644 pkg/manager/quotapath/type.go create mode 100644 pkg/manager/volumegroup/recorder.go create mode 100644 pkg/manager/volumegroup/type.go create mode 100644 pkg/manager/volumegroup/utils.go create mode 100644 pkg/manager/volumegroup/volumegroup.go create mode 100644 pkg/manager/volumegroup/volumegroup_test.go create mode 100644 pkg/model/type.go create mode 100644 pkg/signals/signal_posix.go create mode 100644 pkg/signals/signals.go create mode 100644 pkg/utils/lvm.go create mode 100644 pkg/utils/lvm_mock.go create mode 100644 pkg/utils/mounter.go create mode 100644 pkg/utils/mounter_mock.go create mode 100644 pkg/utils/pmem.go create mode 100644 pkg/utils/pmem_mock.go create mode 100644 pkg/utils/utils.go diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ec1f8c2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# CHANGELOG + +--- + +### v0.1.0 + +### node-resource-manager + +- Add LVM resource into node resource manager +- Add QuotaPath resource into node resource manager +- Add KMEM resource into node resource manager \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..31941b4 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,5 @@ +# OpenYurt Community Code of Conduct + +Node-resource-manager follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). + +In cases of abusive, harassing, or any unacceptable behaviors, please don't hesitate to contact the project team at openyurt@gmail.com. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fd50c12 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +# Copyright 2021 The OpenYurt Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: build +build: +# options: +# ARCH=amd64 +# VERSION=v1.0 + bash ./build/build.sh $(OPTS) diff --git a/PROJECT b/PROJECT new file mode 100644 index 0000000..25830f5 --- /dev/null +++ b/PROJECT @@ -0,0 +1,3 @@ +version: "1" +domain: openyurt.io +repo: github.com/openyurtio/node-resource-manager diff --git a/README.md b/README.md new file mode 100644 index 0000000..1fcf9e3 --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# openyurtio/node-resource-manager + +English | [简体中文](./README.zh.md) + +Node-resource-manager manages local node resources of OpenYurt cluster in a unified manner. + +It currently manages: +- LVM built on top of block device or pmem device. +- QuotaPath built on top of block device or pmem device. +- Memory built on top of pmem device. + +The majority function consists of: +- Initialize local resources on edge node. +- Update local resources on edge node. + +You can define the spec of local resources by simply modifying the pre-defined ConfigMap. + +Node-resource-manager has the following advantages in terms of compatibility and usability. +- **Easily to use**. Initialization and modification of local resources are easily done by editing the ConfigMap. +- **Easily to integrate**. Node-resource-manager can work together with csi driver, to perform local storage lifecycle management. +- **Platform free**. Node-resource-manager can be running in any kubernetes clusters. + +## Architecture +The component consists of two parts, the first part is the ConfigMap named node-resource-topo in kube-system namespace, +and the second is the node-resource-manager DaemonSet deployed in kube-system namespace. +Node-resource-manager on each node mounts and reads the node-resource-topo ConfigMap to managed local resources. +
+ +
+ +## Getting started + +1. Create node-resource-topo ConfigMap in kube-system namespace. ConfigMap example is in [configmap.md](./docs/configmap.md). +``` +kubectl apply -f deploy/configmap.yaml +``` + +2. Deploy node-resource-manager DaemonSet. +``` +kubectl apply -f deploy/nrm.yaml +``` + +## Developer guide + +Please refer to [developer-guide.md](./docs/developer-guide.md) for developing and building the project. + +## Roadmap + +[2021 Roadmap](docs/roadmap.md) diff --git a/README.zh.md b/README.zh.md new file mode 100644 index 0000000..563ea2a --- /dev/null +++ b/README.zh.md @@ -0,0 +1,53 @@ +# openyurtio/node-resource-manager + +[English](./README.md) | 简体中文 + +node-resource-manager 是用于管理 OpenYurt 集群本地资源的组件，用户可以通过修改集群内 ConfigMap 的定义来动态配置集群内宿主机上的本地资源。 + +管理的本地资源包括： +- 基于块设备或者是持久化内存设备创建的 LVM +- 基于块设备或者是持久化内存设备创建的 QuotaPath + +主要功能包括： +- 初始化节点上本地资源 +- 更新节点上的本地资源 + +主要优点： +- **简单易用**。node-resource-manager 可以仅通过定义 ConfigMap 就完成对集群中的本地资源的初始化和更新 +- **易于集成**。 node-resource-manager 可以与 csi 插件集成来完成 kubernetes 集群中的相关本地资源的生命周期管理 +- **与云平台无关**。 node-resource-manager 可以轻松部署在任何公共云 Kubernetes 服务中。 + +## 架构 + +该组件主要包含两个部分， 一个是定义在集群中 kube-system namespace 的 node-resource-topo ConfigMap, +一个是部署在集群中 kube-system namespace 下面的 node-resource-manager Daemonset, +每个 Node 节点上的 node-resource-manager 通过挂载 node-resource-topo ConfigMap 的方式生产并管理用户定义的本地资源。 +
+ +
+ +## 开始使用 + +1. 在 Kubernetes kube-system namespace 下定义 node-resource-topo ConfigMap, 该 ConfigMap 用于定义集群中需要自动生成并管理的节点本地资源. 关于如何创建一个ConfigMap，请参见 ConfigMap [定义](./docs/configmap.zh.md) + +``` +kubectl apply -f deploy/configmap.yaml +``` + +2. 在 Kubernetes 集群中创建 node-resource-manager Daemonset。 +``` +kubectl apply -f deploy/nrm.yaml +``` + +3. 检查在 ConfigMap 上定义的资源是否都已经在对应的节点上被正确的创建。 + + +4. 配合 [alibaba-local-csi-plugin](https://help.aliyun.com/document_detail/178472.html?spm=a2c4g.11186623.6.844.13a019caYIiivY) 插件在集群中动态创建本地资源 pvc/pv, 供 pod 进行挂载使用。 + +## 开发指南 + +请参考 [developer-guide.md](./docs/developer-guide.md) 进行本项目的开发和构建。 + +## 发展规划 + +[2021年 发展规划](docs/roadmap.md) \ No newline at end of file diff --git a/build/build.sh b/build/build.sh new file mode 100755 index 0000000..cd403b1 --- /dev/null +++ b/build/build.sh @@ -0,0 +1,118 @@ +# Copyright 2021 The OpenYurt Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash +set -x + +# usage: +# ./build.sh ARCH=amd64 VERSION=v1.0 + +NRM_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +NRM_BUILD_DIR=${NRM_ROOT}/build +NRM_OUTPUT_DIR=${NRM_BUILD_DIR}/_output +NRM_BUILD_IMAGE="golang:1.13.3-alpine" + +GIT_VERSION="v1.0" +GIT_VERSION=(${VERSION:-${GIT_VERSION}}) +GIT_SHA=`git rev-parse --short HEAD || echo "HEAD"` +GIT_BRANCH=`git rev-parse --abbrev-ref HEAD 2>/dev/null` +BUILD_TIME=`date "+%Y-%m-%d-%H:%M:%S"` + +IMG_REPO="openyurt/node-resource-manager" +IMG_VERSION=${GIT_VERSION}-${GIT_SHA} + +readonly -a SUPPORTED_ARCH=( + amd64 + arm + arm64 +) + +readonly -a target_arch=(${ARCH:-${SUPPORTED_ARCH[@]}}) + +function build_multi_arch_binaries() { + local docker_run_opts=( + "-i" + "--rm" + "--network host" + "-v ${NRM_ROOT}:/opt/src" + "--env CGO_ENABLED=0" + "--env GOOS=linux" + "--env GIT_VERSION=${GIT_VERSION}" + "--env GIT_SHA=${GIT_SHA}" + "--env GIT_BRANCH=${GIT_BRANCH}" + "--env BUILD_TIME=${BUILD_TIME}" + ) + # use goproxy if build from inside mainland China + [[ $region == "cn" ]] && docker_run_opts+=("--env GOPROXY=https://goproxy.cn") + + local docker_run_cmd=( + "/bin/sh" + "-xe" + "-c" + ) + + local sub_commands="sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories; \ + apk --no-cache add bash; \ + cd /opt/src; " + for arch in ${target_arch[@]}; do + sub_commands+="CGO_ENABLED=0 GOOS=linux GOARCH='${arch}' go build \ + -ldflags '-X main._BRANCH_=${GIT_BRANCH} -X main._VERSION_=${IMG_VERSION} -X main._BUILDTIME_=${BUILD_TIME}' -o build/_output/nrm.${arch} main.go; " + done + + docker run ${docker_run_opts[@]} ${NRM_BUILD_IMAGE} ${docker_run_cmd[@]} "${sub_commands}" +} + +function build_images() { + for arch in ${target_arch[@]}; do + local docker_file_path=${NRM_BUILD_DIR}/Dockerfile.$arch + local docker_build_path=${NRM_BUILD_DIR} + local nrm_image=$IMG_REPO:${IMG_VERSION}.${arch} + local base_image + case $arch in + amd64) + base_image="amd64/alpine:3.10" + ;; + arm64) + base_image="arm64v8/alpine:3.10" + ;; + arm) + base_image="arm32v7/alpine:3.10" + ;; + *) + echo unknown arch $arch + exit 1 + esac + cat << EOF > $docker_file_path +FROM ${base_image} +LABEL maintainers="OpenYurt Authors" +LABEL description="OpenYurt Node Resource Manager" + +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +RUN apk update && apk add --no-cache ca-certificates file util-linux lvm2 xfsprogs e2fsprogs blkid +COPY entrypoint.sh /entrypoint.sh +COPY _output/nrm.${arch} /bin/nrm +RUN chmod +x /bin/nrm && chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] +EOF + docker build --no-cache -t $nrm_image -f $docker_file_path $docker_build_path + docker save $nrm_image > ${NRM_OUTPUT_DIR}/node-resource-manager-${arch}.tar + done +} + +rm -rf ${NRM_OUTPUT_DIR} +mkdir -p ${NRM_OUTPUT_DIR} +umask 0022 +build_multi_arch_binaries +build_images diff --git a/build/entrypoint.sh b/build/entrypoint.sh new file mode 100644 index 0000000..1ae02eb --- /dev/null +++ b/build/entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +if uname -a | grep -i -q ubuntu; then + lvmLine=`/usr/bin/nsenter --mount=/proc/1/ns/mnt dpkg --get-selections lvm2 | grep install -w -i | wc -l` + if [ "$lvmLine" = "0" ]; then + /usr/bin/nsenter --mount=/proc/1/ns/mnt apt install lvm2 -y + fi +else + lvmLine=`/usr/bin/nsenter --mount=/proc/1/ns/mnt rpm -qa lvm2 | wc -l` + if [ "$lvmLine" = "0" ]; then + /usr/bin/nsenter --mount=/proc/1/ns/mnt yum install lvm2 -y + fi +fi + +if [ "$lvmLine" = "0" ]; then + /usr/bin/nsenter --mount=/proc/1/ns/mnt sed -i 's/udev_sync\ =\ 0/udev_sync\ =\ 1/g' /etc/lvm/lvm.conf + /usr/bin/nsenter --mount=/proc/1/ns/mnt sed -i 's/udev_rules\ =\ 0/udev_rules\ =\ 1/g' /etc/lvm/lvm.conf + /usr/bin/nsenter --mount=/proc/1/ns/mnt systemctl restart lvm2-lvmetad.service + echo "install lvm and starting..." +else + udevLine=`/usr/bin/nsenter --mount=/proc/1/ns/mnt cat /etc/lvm/lvm.conf | grep "udev_sync = 0" | wc -l` + if [ "$udevLine" != "0" ]; then + /usr/bin/nsenter --mount=/proc/1/ns/mnt sed -i 's/udev_sync\ =\ 0/udev_sync\ =\ 1/g' /etc/lvm/lvm.conf + /usr/bin/nsenter --mount=/proc/1/ns/mnt sed -i 's/udev_rules\ =\ 0/udev_rules\ =\ 1/g' /etc/lvm/lvm.conf + /usr/bin/nsenter --mount=/proc/1/ns/mnt systemctl restart lvm2-lvmetad.service + echo "update lvm.conf file: udev_sync from 0 to 1, udev_rules from 0 to 1" + fi +fi + +/bin/nrm $@ diff --git a/deploy/configmap.yaml b/deploy/configmap.yaml new file mode 100644 index 0000000..bbd66c0 --- /dev/null +++ b/deploy/configmap.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + volumegroup: |- + volumegroup: + - name: volumegroup1 + key: kubernetes.io/hostname + operator: In + value: cn-zhangjiakou.192.168.3.114 + topology: + type: device + devices: + - /dev/vdb + - /dev/vdc + + - name: volumegroup1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.35 + topology: + type: pmem + regions: + - region0 + + quotapath: |- + quotapath: + - name: /mnt/path1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.35 + topology: + type: device + options: prjquota + fstype: ext4 + devices: + - /dev/vdb + + - name: /mnt/path2 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.36 + topology: + type: pmem + options: prjquota,shared + fstype: ext4 + regions: + - region0 + + memory: |- + memory: + - name: test1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.37 + topology: + type: pmem + regions: + - region0 diff --git a/deploy/nrm.yaml b/deploy/nrm.yaml new file mode 100644 index 0000000..ea72110 --- /dev/null +++ b/deploy/nrm.yaml @@ -0,0 +1,92 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-resource-manager + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node-resource-manager +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node-resource-manager-binding +subjects: + - kind: ServiceAccount + name: node-resource-manager + namespace: kube-system +roleRef: + kind: ClusterRole + name: node-resource-manager + apiGroup: rbac.authorization.k8s.io +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: node-resource-manager + namespace: kube-system +spec: + selector: + matchLabels: + app: node-resource-manager + template: + metadata: + labels: + app: node-resource-manager + spec: + tolerations: + - operator: "Exists" + priorityClassName: system-node-critical + serviceAccountName: node-resource-manager + hostNetwork: true + hostPID: true + containers: + - name: node-resource-manager + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: openyurt/node-resource-manager:v1.0 + imagePullPolicy: "Always" + args: + - "--nodeid=$(KUBE_NODE_NAME)" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /dev + mountPropagation: "HostToContainer" + name: host-dev + - mountPath: /var/log/ + name: host-log + - name: etc + mountPath: /host/etc + - name: config + mountPath: /etc/unified-config + volumes: + - name: host-dev + hostPath: + path: /dev + - name: host-log + hostPath: + path: /var/log/ + - name: etc + hostPath: + path: /etc + - name: config + configMap: + name: node-resource-topo diff --git a/docs/configmap.md b/docs/configmap.md new file mode 100644 index 0000000..d69df16 --- /dev/null +++ b/docs/configmap.md @@ -0,0 +1,148 @@ +# ConfigMap Intro + +"node-resource-topo" ConfigMap defines the resource spec in the whole cluster. +Node-resource-manager on each node will perform create or update action according the resource definition in ConfigMap. + +## Supported Node Resources + +### LVM +- LVM VolumeGroup creation, according to the VG defined in ConfigMap; +- Create PV from local disk and add into VG; +- Don't support deletion or shrink of VG, to avoid data lost risk; + +### QuotaPath +- QuotaPath creation and mount, according to the definition in ConfigMap; +- Don't support deletion or shrink of QuotaPath, to avoid data lost risk; + +### PMEM +- Format pmem device as local memory, it can be later used by pod; + +## How to define target node +We use the kubernetes label selector to choose target node: +```yaml +key: kubernetes.io/hostname +operator: In +value: xxxxx +``` +- key: match the key in Node labels; +- operator: Labels selector operator, + - In: matched only if current value equals to the value of the key in Node Labels; + - NotIn: matched only if current value not equals to the value of key in the Node Labels; + - Exists: matched if current key exists in Node Labels; + - DoesNotExist: matched if current key not exists in Node Labels; +- value: match the corresponding value of the key in Node labels; + +## Example + +### LVM +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + volumegroup: |- + volumegroup: + - name: volumegroup1 + key: kubernetes.io/hostname + operator: In + value: cn-zhangjiakou.192.168.3.114 + topology: + type: device + devices: + - /dev/vdb + - /dev/vdc + + - name: volumegroup2 + key: kubernetes.io/nodetype + operator: NotIn + value: localdisk + topology: + type: alibabacloud-local-disk + + - name: volumegroup1 + key: kubernetes.io/hostname + operator: Exists + value: cn-beijing.192.168.3.35 + topology: + type: pmem + regions: + - region0 +``` +The above ConfigMap defines: +- select nodes with label `kubernetes.io/hostname: cn-zhangjiakou.192.168.3.114`, and create LVM VolumeGroup named `volumegroup1` from device `/dev/vdb` and `/dev/vdc`; +- select nodes without label `kubernetes.io/nodetype: localdisk`, and create LVM VolumeGroup named `volumegroup2` from all ecs cloud disks; +- select nodes with label `kubernetes.io/hostname: cn-beijing.192.168.3.35`, and create LVM VolumeGroup named `volumegroup1` from pmem in `region0`; + +LVM currently supports three types of devices: +- `type: device` define lvm on top of local block devices, the volumegroup's name is specified in `name` field; +- `type: alibabacloud-local-disk` define lvm on top of attached cloud disks for alicloud ecs, the volumegroup's name is specified in `name` field; +- `type: pmem` define lvm on top of local pmem resources, the volumegroup's name is specified in `name` field, the pmem regions is specified in `regions` field; + +### QuotaPath +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + quotapath: |- + quotapath: + - name: /mnt/path1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.35 + topology: + type: device + options: prjquota + fstype: ext4 + devices: + - /dev/vdb + + - name: /mnt/path2 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.36 + topology: + type: pmem + options: prjquota,shared + fstype: ext4 + regions: + - region0 +``` +The above ConfigMap defines: +- select nodes with label `kubernetes.io/hostname: cn-beijing.192.168.3.35`, and create quota path `/mnt/path1` mounted from device `/dev/vdb`, and format to ext4 filesystem with prjquota option; +- select nodes with label `kubernetes.io/hostname: cn-beijing.192.168.3.36`, and create quota path `/mnt/path2` mounted from pmem in `region0`, and format to ext4 filesystem with prjquota, shared option; + +QuotaPath currently supports two types of devices: +- `type: device` define quota path on top of local block device, the quota path is specified in `name` field: + - options: mount options, `prjquota` is mandatory; + - fstype: filesystem type, ext4 is used by default; + - devices: block device to be mounted, you should specify only one device; +- `type: pmem` define quota path on top of local pmem resources, the quota path is specified in `name` field, you can speficy pmem regions in `regions` field; + +### PMEM +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + memory: |- + memory: + - name: test1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.37 + topology: + type: pmem + regions: + - region0 +``` +The above ConfigMap defines: +- select nodes with label `kubernetes.io/hostname: cn-beijing.192.168.3.37`, and create memory from pmem in `region0`; + +PMEM only support `type: pmem`, you can speficy pmem regions in `regions` field, name field is only a symbol and has no actual usage. diff --git a/docs/configmap.zh.md b/docs/configmap.zh.md new file mode 100644 index 0000000..444f4e8 --- /dev/null +++ b/docs/configmap.zh.md @@ -0,0 +1,152 @@ +# ConfigMap 介绍 + +"node-resource-topo" ConfigMap 定义整个集群的设备拓扑，每个节点的 node-resource-manager 会根据 ConfigMap 中的资源定义决定是否创建或者更新资源； + +## 节点资源支持 + +### LVM +- 根据 ConfigMap 中定义，创建 LVM VolumeGroup； +- 在 VG 中添加、扩展 PV；根据 ConfigMap 中定义的 VG 信息，判断是否执行 VG 扩容命令； +- 考虑数据安全性，不支持 VG 的删除、缩容操作； + +### QuotaPath +- QuotaPath 的创建, 根据 ConfigMap 中的定义来初始化相关本地资源设备以 QuotaPath 的形式挂载到指定路径上； +- 不支持相关 QuotaPath 的变更, 删除等操作； + +### PMEM +- 支持将持久化内存设备初始化为内存格式。后续可以直接被挂载到pod内部目录中使用； + +## 如何定义节点 + +我们通过如下三个 key/value 来共同定义资源所在的节点： + +```yaml +key: kubernetes.io/hostname +operator: In +value: xxxxx +``` +- key: 匹配 Kubernetes Node Labels 中的 key 的值； +- operator: Kubernetes 定义的 Labels selector operator，主要包含如下四种操作符； + - In: 只有 value 的值与 Node 上 Labels key 对应的值相同的时候才会匹配； + - NotIn: 只有 value 的值与 Node 上 Labels key 对应的 value 的值 ***不*** 相同的时候才会匹配； + - Exists: 只要 Node 的 Labels 上存在 Key 就会匹配； + - DoesNotExist: 只要 Node 的 Labels 上 ***不*** 存在 Key 就会匹配； +- value: 匹配 Kubernetes Node Labels 的 key 对应的 value 的值； + +## 常用模板用例及说明 + +### LVM + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + volumegroup: |- + volumegroup: + - name: volumegroup1 + key: kubernetes.io/hostname + operator: In + value: cn-zhangjiakou.192.168.3.114 + topology: + type: device + devices: + - /dev/vdb + - /dev/vdc + + - name: volumegroup2 + key: kubernetes.io/nodetype + operator: NotIn + value: localdisk + topology: + type: alibabacloud-local-disk + + - name: volumegroup1 + key: kubernetes.io/hostname + operator: Exists + value: cn-beijing.192.168.3.35 + topology: + type: pmem + regions: + - region0 +``` +上面的 ConfigMap 定义了 +1. 在拥有 Label key 等于 kubernetes.io/hostname 并且 Label key value 等于 cn-zhangjiakou.192.168.3.114 的 Node 上创建一个名字为 volumegroup1 的 LVM VolumeGroup, 这个 VolumeGroup 由宿主机上的 ```/dev/vdb, /dev/vdc``` 两个块设备组成 +2. 在拥有 Label key 等于 kubernetes.io/hostname 并且 Label key value 不等于 localdisk 的 Node 上创建一个名字为 volumegroup2 的 LVM VolumeGroup，这个 LVM VolumeGroup 由宿主机上的所有本地盘组成 +3. 在拥有 Label key 等于 kubernetes.io/hostname 的 Node 上创建一个名字为 volumegroup1 的 LVM VolumeGroup, 这个 VolumeGroup 由宿主机上的 Pmem 设备 region0 组成 + +lvm目前仅支持三种定义资源拓扑的方式 +- 当定义 ```type: device``` 的时候是通过 nrm 所在宿主机上存在的块设备 devices 进行 lvm 的声明，声明的块设备会组成一个 volumegroup, volumegroup 的名字由 name 字段指定，供后续应用启动时分配 logical volume。```type: device``` 类型与下面的 ```devices``` 字段绑定； +- 当定义 ```type: alibabacloud-local-disk``` 的时候指的是使用 urm 所在宿主机上所有的本地盘 (选择 ecs 类型带有本地盘的 instance , 例如 本地 SSD 型 i2, 手动挂载到 ecs 上的云盘不是本地盘) 共同创建一个 名称为 name 值的 volumegroup； +- 当定义 ```type: pmem``` 的时候是使用 urm 所在宿主机上的 pmem 资源创建一个名称为 name 值的 volumegroup, 其中 regions 可以指定当前机器上多个 pmem region 资源。```type: pmem``` 类型与下面的 ```regions``` 字段绑定。 + +### QuotaPath + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + quotapath: |- + quotapath: + - name: /mnt/path1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.35 + topology: + type: device + options: prjquota + fstype: ext4 + devices: + - /dev/vdb + + - name: /mnt/path2 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.36 + topology: + type: pmem + options: prjquota,shared + fstype: ext4 + regions: + - region0 +``` +上面的 ConfigMap 定义了 +1. 在拥有 Label key 等于 kubernetes.io/hostname 并且 Label key value 等于 cn-beijing.192.168.3.35 的 Node 上的 ```/mnt/path1``` 上以 prjquota 类型挂载 ```/dev/vdb``` 的块设备， 并且格式化成 project quota ext4 格式。 +2. 在拥有 Label key 等于 kubernetes.io/hostname 并且 Label key value 等于 cn-beijing.192.168.3.36 的 Node 上的 ```/mnt/path2``` 上以prjquota 类型挂载 ```region0``` 的 Pmem 设备， 并且格式化成 project quota ext4 格式。 + +QuotaPath 类型的本地资源仅支持两种定义资源拓扑的方式： +- 当定义 ```type: device``` 的时候，使用的是 nrm 所在宿主机的块设备进行 QuotaPath 的初始化，初始化路径是 name 字段定义的值, 下面解释下其他几个字段的定义： + - options: 块设备在被挂载的时候使用的参数。无特殊需求使用例子中提供的参数即可； + - fstype: 格式化块设备使用的文件系统，默认使用 ext4； + - devices：挂载使用的块设备，只可以声明一个； + +### pmem + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + memory: |- + memory: + - name: test1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.37 + topology: + type: pmem + regions: + - region0 +``` + +上面的 ConfigMap 定义了 +1. 在拥有 Label key 等于 kubernetes.io/hostname 并且 Label key value 等于 cn-beijing.192.168.3.37 的 Node 上用 Pmem 的 Region0 设备初始化一个内存格式的 Pmem 设备。 + +持久化内存类型本地资源只支持 pmem type 的挂载，其中 regions 代表需要被格式化成 memory 的 pmem 设备，name 字段在这里只起到标识作用，无实际意义。 diff --git a/docs/developer-guide.md b/docs/developer-guide.md new file mode 100644 index 0000000..602ae96 --- /dev/null +++ b/docs/developer-guide.md @@ -0,0 +1,15 @@ +# Developer Guide + +There's a `Makefile` in the root folder. + + + +- Build docker images for specific architecture, supported architectures are `amd64`,`arm`,`arm64`: +```bash +ARCH=amd64 make build +``` + +- Build all docker images for all supported architectures. +```bash +make build +``` diff --git a/docs/images/node-resource-manager.png b/docs/images/node-resource-manager.png new file mode 100644 index 0000000000000000000000000000000000000000..d8c5a1e44af5f0966623c11520102c8e40b0b643 GIT binary patch literal 55417 zcmb^ZcRbbq|38j%qU2;GQAQ!MsjTcn!(K;LX145|?I;xynIR(%W$(QSQOMq#?2$df z_x?Ead_Q}>K7aiF`03KQaL(g#f86i4+wFe4U2nJB^?v*mm?O6ZMY?Dx9>-HN_Og=+Win<@U$%V{C*pFq2ger|)rlZ5zA8tR}_{CaqD+o*L$I z5qF6ZSlBrKdD&(S_9cSh(RgG1{Zey;i>Fv}?>|?AKT%pZ;DZPM`vEqLbmf1BCq~d% z!@eV$W3H>+`0pbScwi*)TVEY6{?BdeHxb?;e4<$o|2tt6F&=Tk3%%fZy1%CjW->qLh>5JnDwF z?2DQ+8iK1$OCsefF%yxi`x_(a1BV|DRj#*gb$qx-EsvExN1ap0GFw`^(IT~GaeTli z#og{_+BU((vOi0ps=n+=bbsUEax4F^8uZWWkX^FR>tJbdhv%ZMTIK~<*nKIad>k&_ z&MHe{t4#UPan6bxr|XXWwweofpM*W3<6Z2GMCTg(^bfOV+gTZF^AA_!={tlN6Zv1Y zf1ZoyQ-4x;AWYo4s>VHQqiYNP9c~rVXdf@%^A4HKo*@lphLxXVit02T^!r@HacuMX)iTb~6 z3RQqloPZ9GZ&79kA3wES_LWBk5NSU~3s=FR*ih2<~V($Ii+v9C&&U91G7 zZ>oOykM440yoD7@D8(7T4AUl3^8XqeQ3u8*c_<|o1@?c)`K0@lf2|t(80@cIYgtnz z5=f%2Wjg-8+Z&RoP>I7bMxvMh`tbiJiPEF0ch{8$ALV5gk+E$>cFJ&2X0<*@H&xDE zILIvg?&CSAo|nZN@j2#2uWz{|EX*cIv{K?P^-n^=`}O&Otz0O!iMiE=Wm6Y_Ez4TKz#a6;kzN*KX@JJb}2h1$7 zSPTz-JbM0@?cqoQ+mm{}7m3A?r(Gzq^9a?-$|HR6bPl=+rb6cGhiBmuWNx~Pr#`*uKoDLz?DW`|sQjyY^I{7^a{Z*CoOK_f3uc}24 zea~M0t48NT!`b+rD}9=p@bJF6GeofRp3rgFYTETRWoDpBisjo0=n*DbZP zjSuB}hmHB3J}695vFN$qL+x3fQtTqT_>Eg7-CEGhK9?eTY**TvDEzl(UusA8thw#s zMnS^F3`?1-KgESA+` z9txf4x7kl+-nAeN4@4G=?nG}1K2Rodc}ywI6SlmAwBPiq7+~eERSGFFV2QwDxXB^^ zw+AB*2D`(NT-{rrOR)Em(EVicql|e=S98m~=lb}HY{Tz+7Di+Hxc8UoTFvcZ{j;L7 z@;1DFOmA~8|BhlT zcrS3+I3eTnpxifes^WZKo<`v;OI#V|v0g_O(H(P3W-ZUz_PiqNj+e-DrVpJsIT2UK zwr*GtD9C%~nP~Gg2~AcKd>u5%SL3~$T^e)$J`jCZzm=|}#$JJ&H`3#`@g=c zr4cV#Lcrp7X%QJqjJbDSioPcG+JOp%M|Y5*iQn{4m4q)9GM>epPq1OFUg7V zPsnY=zwHWh;xtwM^_Zo?YFlJcN3EzddRtUxs@#2ii<31^Th2>Fj*HOMboOVl)a2V# zby(Q@lDk(?dSKQ|6uLeM)nkhtFYp1rNhn;UVVtcRiZsZHM{E*8i=(Lr@g;;?&ke3< zq53dZboJLZZj|esJqzECxk~Ksyq%xaLtbArk;;jqZCFOyQ7LJ6H||-o=nTc=-pOv} z5scgni`|Lcc}WH50fyG&|5U+o*zvgY@RK zMp3B>gp0v`6B_@}>v)8Tu&^+Pu`}Q|;B zzE2a!AGy9n{L3`yt&<{siwq?5;T*wt`7a470#rKi^WbL!nNb1qAL+6p?xHdBr+` z^-W4ht~Jge=vPsaP`rS`RgM>wj07u!F!SPR$vhjh$GzjhLw^ei6c9ky>v2MmhwLK& z80ULzx->BB69pRFb#XAI#P|ek{2fF;xTntZ+z&6z=VM^c?HV(g;z6Gy96(;<*XEZG zqXY8^MxiP3=*5BD{1nj_a3;yeoo{&xW?=xjWGPnmZFzqF%PE>5c#hma9OCUtf-6NR zE<}?^V|$*v$G_wNWguT4L3odve~`KutasQ!>osBxXLp^f;&!scpWTlLfeZ*oCimzv z4dT~o|BpMEsr`AWX@@PLCTBZ7TAn8du=Fl7?%&YrBa{j?-0Etkf+U3d=e83BfqDkU zo>ZEO4gg~}_kT>9`14c@20%Op(&%ham`NoALl@rN`Wx}qLPIy+?R>)oE=znd$$c_- z2zyOTP6)f~9%iib(5<(a8uoxF8pfvXEfPznV(E?|TjY%Je#?1}iCN)8x1lbO02FhT zTb-NdHc*tGEt(VuMq3kEjJR+k_NyL{8P~HlQvsj{Vi6#-wLq6SH(76zeNB)~JDp#f z2`tBkw2WC6jWoD|>Em7YyI`c+t1_-Wh&V9Aby-R5Kg&a?=kko3P#_b%rG)!~WO zynzm2#LTci1L3^_Z`^H+(!oe{31pkT2mgGC`12Hubh7 zcDi2dZlb<3jZk92m=G422kh==hx31)A^8?O!%mC&GV}(`C7_NwPr^(Mf^m1LFRagX z@7R`rnT%%qrG`C+K@iUF^R*XU<&emvQ|(7nvy zUa2R^ijZtj)8`G1jEoqv9n|bPS%F%%iIE~0%LgyPdO&D6&yiG^dFFw6{-Z*S(7b?k zO=MvN{Ph;N^aACr=zp>l9C2Xh#PyPl&{J+fo6xLap7qbrykW7FCf14Q}vyjU$6rN(cYe&yzk}#z$-X!bQ(a*Z3mYPu}-YdeA}9d)J@YYwn|eL~ zSPV!7vb7Vb*J5s(n{Ezu+#WDloc|QB+VqzCQoyyhlMe%~d~6pvkR=jYL);aR%E>a3+egP5Cb2V@i(EF0g^z!4?Eh-IUg&w~T4*)$xOl`VChbw`IFZG(JedEu?@&lIk;xa6Oe zKgm1q~OssB^g+`%ypM0OTb1aZs@tCJ$Osh^iR{)0t_I&D?BJjM zko2=P#;tE>u}^`sYB!OLO+8_MYjLrW()v-lYNp-tPOS52YVGrLZdqFkJzYRSSsdDB zrr+P+c3p2~6n^D@nbp-X)^mSe$Aw7rcTSPTz+|Dp+>ei7;UOI5OLwR}Hai3t4<_Gm zR{YW~Fnzbso0(2&RU&qd+CedEc>)KrabPTxAAmY0*_coE7%B{2akRha3N+i3_yEqy zotuF6IzGzd9PUY1Q)`bC-s(})b@?cf`M7X%&^$X;@uT?uTvAZQapg>eW<^b%xNn+D z`a<>bVKy13&Jsv>G>f{KKYd6XEHro1-L-UTVa=|X#N{aWDjie%#Evyk=9nJMZI<{j zkYe7SHS5P!lMlJ=>q{m>#nwA@*l_%79-B&>SNM;1r*?*j;5r;-b99w!b?~P{MVWn6 zaG?lx&7Z!6{0|wV!xa0nv>oR@zIs~!l0c_K|4AsjO5Ux8ZLeH}QTJ!1GN1#;y7P>m z?PuHeSISp6zj0R2g(`^r3fOjbKR(=X1^; z5$M{bs!z*4(DV4vd36}vH=ud6(RAJG+13D$YIUsnlkSTWR1<|g`xeLMv;^l)cQX1R z*t$Gz-99JL0f}Q*hjJeC=z}w}ap&07`&@)$S!_S|aH&4r9;)28bov5ByX;G|?K{Hz zrjT=s>~8zsFHO!YJWb+rD~&1 zP^ZL*%Y13Kuli`F^=@wP7Z_=JyAS7Lwyz}Fa6cPqOoM^rkSrbCc_}CAJ{gY2t^t?M zpV}%75AY?awkc#M))zt-d?}&7lL*ef1T6PCpTQSn*8XFW-SOuF+x-U4o_iuoIrR=R z;cBkBotC}+tw{c#?+q&AhrXXJZv^YO{N!U(+$&$K?b(B9U%TbPK3CLBTqf0s!*Xh} zxTW+8u%c%8?Krm;eojdsaq&xVHnv;+x}t1&{>Eqhx0;Nf_dm(!e=D>uY5nqB+^eX5 zWQ;J;MnN!RvFGa5gkkrM);yz!MPW>{If^r0#|QCzU0>WYg2<$2sK|0iSyjJpq1b7jKj~udW;lnUuc9W!g?2i#RtBLJ~9cfDWT|Zn`|kvl75S~ zBav$tu8X*v*jq}4E|VL`L<7|b7Hu{fFAJ@Iy%Ei09>I5mftB^!!Ct^su8NvO_r3Mf zZMSx;2IDIAc(0>96He0H`o{W*z5a*!+sBq#78*f?{3L$QrtBvD7#H>zvv-V%HAeU? z3vHT>6Z%Duk9MjRbKgyO)~XDoqO*n4ulELKT06AeDTYIV=y zTp=UumnDHSTeK6!Feqhd@moJvgWq98q$(x7Eavm~y;T(7^nu^LfFrHmdililynyD_Bb zq>Jr;IJ}fWICQCjH?pvgDCW)bRfF0WzT~P|t#4hxsw%G1by(1w8x6@)aupb<@WwRo zW){xe14W|cPo19>u3MYE4OD2m#`N>4TcUcuzT-@{Nv&VF8D-~S2CQxIcOmTr6*o(1@<~x%|EiP%Y=)bb`*{?Z6 z#3U#SN%troIuk=?}#A+3PVgB&|>KRe-M)38ds}OCKAwMoFxWR#p~o z^dnh^%o~PVJ7A}1LcOTLi>W=niXiP2VKgnhp9m+LqCb`U^rJgJuzCRSSEK^>2 zxhbo4z=VNw>b<5*QR&lKA6#b+=Yh(s!>#(3*7J0Umhn;jH*<@)P1@@tSTfaf_2Z2@ zdf(Z{6}IcVu@Ljqn5801P^VlXkl-ccg*S@y`9?Vjg?VvRRwP)x`pXevQAn$j0P zh`B`{y2bBJDCQ3s1C$#OFvd0`pxnUbhv^+~Be4#u28W6F#k<##&8_U)HxcK@T4jQO zY*0dQa9R5cQg1{Y&Mx)$OV`OU>qD!8L-|C5)gk21{s=ZN$Z+e+0zw z!VY-_nis+j7jj7kP--`Arvzn;-iQP!ss22H=DNxRa;CF{6mn4YvW^uP?r83(S#Sv! z$N{FV{ofM)%2MzWfHtwtg>(Fq|26|{O1))G#;DiO2Yh@pfW-2j?wmW=5|eM9BwWOd zYzAEn7YzT$7NVHI7V=Aa7=tlok{jT!D-zO}m;%i}oQC-QKia3g!gJEW3%&xft}l0y z_zLrDFQZ2+5e)EwvcJ zGgvY;#*Z>O{E$t8Xo_W+Kce{5c4)L=8sMinOK|=&*bQ!Y%-1n?h2qBlzTbn(nD6Bn zXzE(#uzRml!bAoxAsI;!o^nWY*ef$w(HPsc03$~~G7V{bsr;u?iz5YWUNT8a_v8V1 zA>O{U9;+cQg%({^imJLZKXot3?yOon6w{@ zSjb0Cj21!Kcot}YJ3*P@R0Gp=x7K;c<1$8axtqj^V;n=mz50Bw$KMD%jCWWVLcXzg zKHT%4@3bb8@v2%nV)J@Jp4cyn>yP-yp?rZlQsk86CuW7zGgSD)1@J~7aV(_X;xZ~% zJ2nh`Cy8-H@*J%8c(}#J#M*sV4r%2c*ic{l1V=V{O?x6M^59B@m9)%qpD=kokQrVP zFv*;z$kCIbgZW`a?akT81DBnsMwJETLOhDSYR;^LSH`YHl!=tAZWwPm%DAN*Pq#6T zeZ^%yrxV7{TY#x60CVi*_?I&>+d=;+q7bLo>=KP+gw7}rho943HjX9bs5}mNDZ=Ps zAK3jXWcv_NTa zMtNk&vC(7PRWbj$VZQu=!`m#0snJLg0hy=!3iI)TEw6vJT+A*>7XJ3BG@p`c${5}K zHgD8)VVvsJ1L1{%r6;@$Z9DN%AlE0;>(99K0PWMDWka$!N?*i1+j;{npq&gqXc*lS zs~iakBltF4*xGI%b01tZIx+>PumpIQv4vkEK1_%j?Z zr4B6Y`12Xsb3UOLv&&wpSk3!{zHDk5`_Z$3W;B;%NR#Gk52~grx7*SJc2mW|FVS?3 zLy}pG#oEaJANv*u_I+A+0a{vZ+3dCzf5XD<9>^-Vws!0 zj*NXD=e0#+bbw3Wq1(GWj{EaAn))rDNKa+=RUNE2fLee83M?QDx8oWg|1^&MzTd!8 z(41=3E9KkhXZ|c3$??9_Bl!`1WbyVFWP0Ir_U|{o@9Z&-jTkQk zs3Q1bXw``-#?PTz&9!aTo4B%gj227BzwnyLZT@l)Hk-LHUzUm-*!Sbmj{cpjIml%a z%vvsOPLAdlyz$FX=@0W`_c6dBef9UIPImLxTc~&0O`2s6leBD*nR0f5n{BzB-PXzWRUw3TIZ7Qh=I_NhinT=JW^rgn7;+)gs6YHJ zOZ`)O?7T_*zVAwm9zBU4S}w%@R0{cs>hfe=Hv!b-Wo;X~A4aJc_cmdCg45G|BdqU~ zZ7qiK7JHgB`AR4>b(eg??9QPRal8M37iOq@di!qA$v!4#LjAg4s=@f}X1Nu$dpKOd zKTJxSuew@_;y2pik#*`wYifGm)5pbMaG6!T+B#NllA}q}-wzfko)ox2RZ;~QOlk$` ziGBeeKvgz$f0QB6ZOk#pSdjcf_k8p|;m~*T7;J-&4~kzY>sI%eVux6A3$F91dDZG8 zo33TaH4h#z2n#2xp2$#~75K2hkDsPaoa7f~Juy_PfS*MqK52N_Rv2OB?Y24+n}Icv zPBpa>hI@<^VcxI}#V-}KRu5agv}@5aq?Z#XFY~xCg~^fwEqAka9yz6q<`R{%*0g&Xzx^DJG~RahLgq%nk&okM zgSI;`NDb_Z*BW5e#&}*V-0%x))>Qc8l5zCJ`6KW5JP6myH;)!?&Ffb0 zCdu(<<}k?f8Z?b68cMVha!8sztEe87GXE2&)F7BS;4w`3?4@ISVf%-ZnPR~@@Gp3@ z`7*|+ma>)!(KuG|4W=vGkA~QGZnP%j%c0YoKBcs&o97xmNI4H|BdQEVAt#|A2(c6&YHaAsAbbR`GJ&={^dfE zb?7xV8Y3gA9jl;;I_}t~6Mnu-WGT~BMa>zw%tiBTf8zNr5{mKj5C0wGztG-VAOK-7 zZkEH!D;=(|P9BhyL#n%O%5PSWKi`{-%w1>B5^7seZF2xw)cQ5SCC4VR#Qw#gWuCA9 z%u^7R!FE6yGkr}eyCzz}J7ic4?Epyt<&DoqvFsDttXoRhD_Doi*fV4-|0>>+i9Mc9 zn@fphjLCk`lD&|?m|ZfLz+{<8wbqwu*{UVO^T(+G|3m9qQW>9hL+f4}+-ex{30=A{ z{O&M(yK{If`Hh#oby@S5dL0)F8Mmd&U!B4PYg#4EJZx`nzvMkh5QrtActFS+D~4R({!Tf0gdVH~ zhGh5Dc;`*;>3~>ZK&%UN+NXxeg1X$LZF6T^*x5Dkp0*8gXYDh`&azazE?c0xoWL)m z)GdZ8OJiAr{Gh6B5Jn!!b&z7>@+o|vj+n)h<*liKFrPqLyTW=q%54w-1xn9jQ8@EXyPiBT$t z=_KY%0funPk#XZB_24|qH|d?*X}MRKjTTbS=O3LG5&JQ49Tt`|*0PB~d*jwHUV~bn zP|@Q>&qt|IaQQnPLBW*-Td&p7oE+hVzat$w(`zfQ_OjW~wgj6vZ{^log0P6>Ejdgv zIkEa(1c(#oo+B=uY*)#Wnjf>;&+=@OSm;JA`lr)ppxMORqNoG$W7dCg!TQ z><;hD=i(zf@;WWQ)pxB5y9xZ%CP%YfTl7Sh=fjZ{RF`xvxt<0H7|8EPUP~#R2o^1? z_v;GzhiSI4&qJ-P0;!&oEz#rY{ztJ2d6DZQ`6?}uV`_3TE(R|}W+S#U;uh120-v)> zrrMge#EAQz4DmN2ucQ5OEMitut*!ee!ujAsL8%ax$Cw(Yn_9?H>pecsFzH8x#lBl2 zd6s?YL44Wv*UsrMInL3ln7;T!VEqEs9?nrjp2lx7lwW+A2BQpP$wEG-bG|&={qqOE zw3KJ5GLPTAW_SO2y1(^c&#&1}G#9ZWkHk6d<_7BXAF-?*rG~Wr;GepY+H5wr zB%^n9i|>X}UDHYciHcUR-Pf)6r#WYhC)n8~!Y3xZmR_?NuVr(``nCDT%8>rL&^v{P zgOi&@i{Dl{W5!>!GF62NhIjhsD{=S*x znF^PGSv~z&H5dFs5x-O%v7avN;Z}mE(V1}X*FJh06R*j2bfdVyq&7Y)oM*7~Ds!Ox zrhU5EDX{pf21b)}9rc8>Y|XIV&KYbsG{??8vcQ_vAQ{nNI%?M>hs{eJTDAP;@ct!g zc!H{|v)ADGbq>kYk@Jivq9YEz?_Fnk5>vgesI%+vk=6odLTiCSJr1Dx`xUd{yLVBKkRo_*+E zl{f1o&*FC^a;}vMyywwT5n%bn->j310DZ?XdC_@k>RoQ^FO7QgK-+w!#-nzg3{!q` zbU}5G2|+Ud$*A-SU{sf)u#*T;GlsiA#!2u@uPO3J9>-lXdj_N*aes-lD>~p&3EH7) z;eMFpRX31tH?&Gd?PnfH4Y+bh^3UDLII%pu3WOevlrmM4-sGLtcvB$ZNd+Xj=1cpq zT`a?t_P6}<@-?)&uk4~<_0WMLt=SOSd{!V5Rd-}%ezUmESV1HB&W>~6o3wLM)NYb; z3hmQ69Zn~Dx4Znc1^jHGh2nxY@j=7>uDBM)4+T@9RPxbP&Qb=Wwliv?{kV)<{fhN^>m*c!HF2qVn7T8bB{3O zP+}MgBz_s8mNCihxXD(+16*_i^+)0|X1}h?`@ciZs5WN>uG`}rJsWJ5)F}UPC>7cT za=A=`hbLp|(%t}vP;RRne+cVZ<3#88_++@JT|Vty zBx8g$xTkl@&ZE`FVzeN6lTZG|0 zp4rO6D-+2n(J*;0g{Mx&(Ug}u7y8)k#9GEbIbj!6Yk=H_MO6gNfRcgH&;+MKiX$8>xXyd9y3zAJE~v)-LYyEsug@^-LWrD zoMG64^nj7IT69p7^+fTnm1u||KEux0xF;h9OTwQ{&_OUcwq}Y!zI@7#V=}i;ttE+O z=>r|db2GqkCgh&##dyGX)(Ylo<~V+q}^otpf#gxJ@c^<7ICru-BPJ1qp@ zI>S{ku6&6kC-4b7NFUpDF2KS%ne}3#Je{9+Z$F;D-j2>^=&t-!)4e(z5TEHRBPVY=bk__nj64b-qC+=N{>lnvvHQfzmUace3*%LoC^eLY;9uIQ)4Dd1A~ob z7ptk!u2BL8)T*HHwlIi;6f@C})04V$Upds|R#^)WV1hIr#Lp>diXX+S`U_o-VFg_N zrsvPBw5|qRMxLQ2B>wD8Wxc_j&Y>pBt+!!q$FxyAN=`bsFDvIjD2!f~&YQ93z2Q6d zsiMVB7@8DNdH)#(zY=k96gH9M;i_{iGqMqv@tp4E;5OXPl<=1_oQ3bw3L-d; zxe(NF8#T8306)d3mdh6jjo~t1jyYS><#0kkQ3jFvj^nf`A7=1%+?yT~#<3u%F9^&b z$>T_~!$6u!>x8YTy5&Hf`bciRk(!HApVOBW$)02D3s*WTQkHH!)YKTU9hMQj(5ra; z#8(~ws{6!g_|j7Vck9(Q&lQg-q!rrMN*yNUtmoJKbnL8W8203L&f2k;F&i4iQEjsk zYgvmNR@fj96X;J5a?5EVcD{g71SeEwynTTi!=eGKh-%S^y3kC@WcNL7o6ZTRG47z3 z({yX``kfI3$e=>x$4MKb#G1N`R$gJY*UNf`0ys5$4Jdi-Pow=q;MTb(Pl|n@fN8vG zU)wxfo23LQITOHbi9Ds3YgQR=e%!gh-?_z=e5cS}IQEh|+pQ@RaPw350GoI`wy9x>IVj-=1b0V^7 z(aY7_9f|$8H{B;Aw!dFr$jTh;#bpju@T+7yVagzAbMr! zxNwDDPDdD~{&qz*P;c`fGif-$ccUtF;W~AINC%wp)}#us#`wvjk!||CL#3tg$M_# zps0Y6ao55r1-QnUnhOg!au`Va_o;(mKGd=%fQ&lNX zv7X`ZUAdFC@8FYm0evo%PZBp6qbYR)eeEm+9V-b)VFBFPr9|GTK3%bdC`t zGJIe-NN?Gb%A6UQxAzlrB5EklcGMhOO0vWC@e(^tn8s5iG36y zRln1Q%bYak6qvt;uBWzoh<}>Mo(I%5DJ;4N7)#n!ACEkj9KY&Fz^3(B;5TjRGLMMW zExB*`H6=UAL2V(?N6P!N=Uh16!tTiGwIKiqTJOu6DEiPOPO3S2+Gavr!}(X6i-Xht{k_Z@1k8+fVmYV&oTOui~LFj8>1Lov6W z_kAHv1Mi2KQ^ZIJO#JST5*!Ga)-=J=XUmEg16;d!@}3sOIChR-flQa-kzNPWe8ZV! zX?8TL1Y?0I{x<}+W~YISn}uHx^>&K^9Qmxl@hss@Ow*&26(ln{UQaRgR!MZtI3q}n zjGl*uw^v5IKO`by{&3YOGmcSM;~tBWN|}qNaY2;oP2C#aC}>`>g-O4ft&#>7EJJnq zcrd~P-3m|<2z*FC8S(3cE$~8C&)AFn!}qJx0#?eK>_W_HqK3o{eV6`Kh~#3z10KGQ zTJyC&NVa*tI)PnCUJzmZniSGD7OWH>BC5@}BCjb(y5jbVz`}&yl0-tjH|l1NZ8E^B zE6=+8J+Sk}(ev|By_W`KOi??-us%aEp#{+dND=XD9Z1PTrLr) zN`p89%Cwj8NgZkZ*_U&#%AW4VC6IyCv7dxle%Qts#e zFE|$;FVXN~X?XUj1wj0w~ilWwzJ>)}AN zs@(W;%Yho~LA)RU1D-WT$3e3@B2(v5Sa6HvULfdeW61<4GWqQBnsd2XtSGL7u6%`Y>}$x9#MBJ*~~bhxxK=2OZb9nJdyBu z(ks8iVjS~~Dm^G|YyIScW)Un7^NS8GW5jCiBrN z**~qz6RZeb2a?ru|M%CQ-%u6jZ=LynpV`DFhZ1JX4|MM&>BpxY?EJaqG$#jLNdl-c ziufdrS!E@hO}9#`U3lliO6lyi*TieSH4@i7(15I$cJCEol_^mmLQfYaBwES`20~3k z2MosEQ(y*7Bz-U7m}XVJ%It$c(&>AO_)mKaw&)>8s=n zak3b#7!dTT@x~sk@+i;C&#xR$diHB_|7VOfx7U%ow(ESd;DA;2q0{OZoQ!QwyRXPn z6OcmPbV@(PIz`Ki>=~|(iSCY{VOP&VI4*Q|K)ge)w!1gxHaD%W!z$f&R0An^HkXUX zTz9%AUR}>h&njz)iRCtGAq7`Usp+^Rw}dfIh2MJG%_Iy+aF6p0{dP>8R-U|~{&~0r zXq3H|6ohg&hM&~GoCn;$M?u$w4Y#OlYOwc@4pw;#8ZQ)DkBtgeBa@}e#(m%wWy>P$ zDrsSWM@T*MEi&l#y>U)RzO;~h*MQ@CVUjTq6{2z!P*R~c=ZEdvx2l5U0M}~tLa>O( z{xeiFd+{*c!CoNW=;!O6yJEQbq(8ilHm+xCu( z;BT%ZZn&ni=i7y_5?0XD&OTzEn;M1MP%>*9*v1au65t zn%mNC9_!IzxF*{bU|SWh2CBWhJRmZm-R<)kZbqTZJ?#T?-+)o8!vx zz=3uSvF{m_Fzz!?I|ayhPk=pfzFoab85pWwHFk?pq;^V`$AN49eO`1oe}4r z1hF%#0&sP!l8Lj0rX6($F&;6uk=33at0v}sSzE?<{hk-A0dLjwOqdKu=@+;@AjkgH zHn08uPU2cBpyR5_ip}0^y_lnyWb`efY%cLlg=k)sOuk9`w)jOsAR6+@y`M3zKtz@C ztx;SOK_dHe@-?60Tw5T&CSwK2dlhAq{;U<|jPf6tVs38%KC;_!8_Bhm=3yK*#Kc}y z42dWN5qxO>#YL#Sa>W&jbZBHv6;}U+cOyI3=KFJ#__)4AZ1g{op|g;Z1ANR12Ey&) z-SX?RAMD@m6d&&u4*uUe&}`K;qs zN~Q=`$1I!QjtG*iYm7hPQm{x-ZhP|Wn~ukpO69Lc%F?q;<$d{fGt!*jDY;FFvHh#9 zFOC(wo&99}(&r^4c`1mBC2;#P^!pL{>lt_rAP1wfw96M3v#Ym#hy*PZNMlg?y6R0q z*RyHwy?pk~eW}05GG@oO+nBIFgbGG_JVxYc`h*SJe?*hlyd`32nu?dJA>zeA71Qx< zljtVcG-*!DgQG!~dv9^Q`DLDgYWuCpsG+lQ@lk?K1<>)9*HDX1^>!O1Js1=5H*#-4 za-IW>LbaraffX9ePVRt;JQ{K>h9JjgLfo3yj~$g`B*2^A5TVp>Tu_S3#9Lnz;p38= z_lFCx3)u_yqZFB3ye}jP0f4)viEn(c`Y1!aLlbG7j|N=JtQ&9d)(xpOMI;V*GB5UK zZXEUL$Wh68>-P;VkqvdfKZB|~zxI2&3UbVKH-JVPUb2zcetULOk=DQTdJA5YXY{Q? zPTw+AZiuVFII2G&^h5PTaU~O=V>7CJR;(TRVn6-$T^ZPE_5_L%qtVp;)$nd;^Ht4fao=(KoWyOeM0k0BS+rG_08*(WA8p z{DHIioE2;0ch5c7Syl6%W#oen*${J_aI<#q{+j`^TV4bFdV%9&Z>%-nz~Kc=JoJ5#9=Z_}Rr>V%R0nBIAiXcc5XD}z!Ij?>1ez^~X8;D(32tvf z3i>;WC4vmskHo(V@?te`ws^X&N*RjdkBd)R36aHd$mKTFssQzc&o%aP$;p?{>5bBt zV{i*X9ay_zH+0^4x~V@FX}!&T_u;~`bM4EDW26q(K)-+)Kx|p<7KmeX=RwWUnc+GN z{QzC>CjDwOm^t%PEZ&m#c%gjfl5X;w{SDs`H~N0qb61(_tq@)|`$G%Q2so0Z7~iRS zo*Kv`SRW(-q|cS1-^W=|0Q()TgpdW;v_5tG2zViRW!ceE2E?yr#t)T~{o6|Fl{JDB>|`BL+;8*L*<>{ zoVO16Kz^~`P*`|MN;t9>WWP^y4K;Q>(zE1&z~;mGImSt5lsWNNcWk%qArUp@Dq9a` z<-X;hkk;=bu%!h90 z2zeaG8M|&A>?S=%{8d|TJrcC8KZvaivt$Bfyc95;J#ztLRSM8C^NWxn(ZB=TM|>I0 zf0pYwm4ILN*3XspZ4zKJ-kLF=ADIF;F}Q8ySMh2;4Y3mnw47*+nx8Q$6{mN9eD&z~ zYtA>!@iQ7Vpm#05yAZkHD+1+v1El#}M&(CJ9CuTDZ=Mmn7kcrvvfk+R&Tp@zdk7;q zC`n9cwO)drzJ(~rp7V`SiwW$D1y`B359%=KNe&o~)C!s^jVbJKCBk!g-c_X{1TIfl z#%)@O7R8(9$kC79ls!{CTffz1FB;E9j2BZ=#LfZLoB%yxw79Atlr0JIV$L2yI5$=cg4VVp$bXP<)& zy}o(&JxoPRFma5N;)t{9VOQWX-_W(k!+ldaz(j_gW+m3YQU(={??OEt=vW9%DDIM| z1G^AJOCWQs^{dPa2hF_?HB&w4LO7aoRysK?3G^d7@Iar|Ck*tE@?})*X{CqE6HBA* zp}8UOaeneN@(KWKO7c}HQ5j&uno*n-&sCpL-1-p(P3>_&2jW>1$3l(Z>XZ=U zd?x<+${ZU^30`4a!o6WK?-P$M8%eARxO5Sr$A_&+_mm5{L}2aIy)^C<_OhuSDkZ;w zPH1r7lasiJ013~%I6zn8&9obt-?vWf5=PBY#3E?SVAtbl1CJF24uw+(qNLJa`dq~G ztyvS{kksJsa@_Hr0`0}e)Z!f;0-#bp0r7};cDvkw36i4t&w!G-#u=?Ql<||G?`ays zESwOFhgKM0Oh-TAlx#YRcUbG0H$TU7gP0zL3IZgp?G#qQe{&N4`34Gb@E1mxV-(xH zYGnr42=|N@%Er-q1K9I4BM2H?nCRGz!}Z|q8PC)W59ZIoEW~(z-oA`viR$k|?E7!# zlcB>5>wg9h<2qbI{p-x9VHnu3!x^DG8(R58V+(bm?zmiOl-y+CeDUvJ1)C!T$RYD& zFQL%wpFgm0V2pa-TmbE#eKogU$N9s?f`J>s9jH76YO(*v?jZX5^#=h+{IY8jm8{|c z4MFg^s{|;*JQBTIadVRjRe=}aBC$U$#Am-);Jkq(>7(J~sT4Bos zRh&9{y=`;yeiq&oH3|(Np4=;9yypm2@RN5C1$A>mgkL~Z>ht6|#boNw@;qW5 zCA@~qt24&g7~|hZ#PmrKMF!)Og|BE40U&^u(OU}z7Fpb1m@qF2W}24S)kC)cZ5n-G zl73MEaA;c!0csGa1cIuIIAHwLg?1LMD>6j_6?ozvDa#J#i7n)+vOJ<<1Bb#Qy$`Pu zF3m)2TMSJ;e&Xga5D<39v8#s#GiD+*X0$)pSkRdLV$Bgof~8wQ*xMw-YIWtr+Bu>0nK> z+}d;@kF(I4mtVH4?%u1?;+z70NirA_{~Yam1HDp@P8&GS5(erZ;0U8kW4V!V(wN-2^c;r(Fh)yy-PBVEOGa`+YO}2d|0ez-d%NpMS-F6Bs{WF9QJ7Phkz0 zx2P8qv}pFmWn^z7aPOO^fmM<2?oyd89ix@Ue(wB;Jaa#ZxS#gc3?(RqZd zzC)ppEAUs3QMXr1t}?%ADr0@>>?Gp`hzOSFKfD57jb00gaso{XzRBq0Wp!@QAmQ%{ zT6DCpmD;)|E+gJ_aV@aM`yBdcM+Q?*pNlc>AO;TWf!>@whh+1fjNd~>;CFd;EoMf= zuu$Eg7Jwf8Kmi{48aNu(!@MaOjgEB)?QPG9EirqD({SM~$@KH}Nm^!wy>%1eIzXgf z#7jpQ#()M|e5-k^csg7=82M32U1bimThLMI~PZpMny{D5ilNeRD^+=ELy zTN65{%~Jtm0H^~%opA*T#VJKV7Ztc&Ko5P#00}36i_es$fk`LeU2v=QWu#Osx#n1< zuwdt<=qe#fj2DT>Ms}v~bS?u9e{y|~v+Engdz_=+*?K!-5?kNu6%!v$^O+U8Y-5{`(h8K2OIS=^#; z4i`{A@O~F(a(jJtnAh9+$p%h=;8oT9i(fB3n^iYkDDLFfl{!cD3LO7YZ7mRHbL&OeIF_Nta#XV@k8CSklp<2dmw~RV85QZ z*1vUKi3~7*V1U1kmsqpIC-po&?Ck=3VLAV6VYC7mEVCvTLgWAV<#a@_FtM*GDBTC7Z?;M=7Y&;`rIUjs!uMO@Ei=B@G}52=N18$~y%YT!D_ zuwy))inR=nji{&)*~z#?$6b?Q{s0cL(_TCwbwFk9o!|hf5nwyJpqB}X+V7o*w|a&E zJ{i6EE4)bsI5qy0n(@TZ%e@zocRr0kT=@`A;OT%voWHN8He$tLF;h(xVy!@cXBZ5o zd6uBJDmvQH3rpbLB!{7zjIHmZSvYE=Z>v|ud<9wi)!0%+OjtMV(tkobE#zhPO;eFx zvxUp0_v*WE>aeVq(THkp;M6yKB>g&V!U|5WSPdk2dNbIY?h?zSWF5Cym8WI{X2IC{ ztG<#||G1=CneFc1za4+~^^+Deg^JcBCQn|yni(d@`L(_2IWN%M;(lTwTPZFIsNC*6Cnyz1w-O>yao&9+<%tY$8z+Zn&P$ z1A0UwDCk($rJ@^Ouj@5ae8bCk3VE=YZO6w3F^~reJOo*|-vP!jMf_y8Q_4pkexhU9 zBdj+OfhL9p$dw9&==I2e&^YUrI9#>W{$nQIYi)#ni1%^tei52oiOUQx{0djH`k^cs zb*-;eCpb@b#vl>Cng9Rs_7+f8Z{PkXY&NNMhk$g4QUbzOx1Z?!Esve(&A!o{n+OIA?tKirMqC)|^Y24Og9@5~i6^ zY-o5uzlXeD3JmK<463SuBcdx~Asac;pO0%B!9}t1ce{<+N!Ol?YGY zeF?U1 z{94Mldjr?Ta!<90v5rS+642PMGh!}`DhI~Sc(^7@AxMOEd_!232Z9&Ao_y{RPw5{P zE}6jvLYqS&Wp96+N)}I8_896VFaNG;^)=Y_2h`o%Q8T)q^|wW+dI=IPH*HUqSseVh z%-eE*Kff(H5gkrfWda)O=gnUN{(%cmzZ?>E7iJNb zPStcaX1HlS8Lrh92DKJO>6@S1bDNd!9HE8F$68woDPPCcEJ}+`j1wt(=*0Qo9XRO?~weRwo4!& zuua0vH>8bw>H8|Uf+4bMImk1Nka^-(sg6gsZ}a^7R8f|1pd54O%^Ng1M3@|XBJqhD zF(oM+2~Wsq39N(V8CCnfcw#457rG@*?mmmxqgfC<&yAWK$avzmHf3=1q|b9xsN_m4 zlEI+D&tHEBo#F&-=Q>4w3V&%C{&k0;PH@M3cg5{#rMhj+u9fTAk38gr54-+MSXz~| zc+O{Xnx<1JV=wml8Vy|FHuKyyos8o8`I2wpD>3_Jwl+1tgUwc;PvpB8tEaq3{<%%z zzgJ=e8ni6>%$TGQ76CindCRFdNFpb5n4?e+pSb)p0uL+xB!eD!92zf$>(llx;RN}x2oYR*kcg6R()WycTLZU#SD_T^WkF3tVq|JkrX zlz8~1IwMZs&}HR}TXTTdiU7KD#^Hd^c6J|#1%Z*4AgKEaj@tF`qw+AHZJ|>*SRX~7 zKF*o&f(J+R(Eb;Q5+@YlJ>j3H{E;uee{0!#3 z+JbFRRV2y$E&)#|aA(e8Agd#r*<0zFV>Etnr3^DsE-+rtyDFbvxI7j9d_}u%mDR)I z{28b%00m(q?|3KEqE>I1f<5YiT9jsdR^*Jbkob|F0q?fWXi)I6%EoDiPWjrcmbX}~ zYp?4$*y1|XRz!pz7Ylk9LLobI|1>u#HEU6_qnbe@-JuWKm0ti_6-L$Sr(W~jnCzQ^zsMOfjx$w zFBfR*MmY|6W?_shkUL&ABw3b+_-S7W%b4t}x$I6JIdBYa@hczIIB0H@&sF8#O{?AO z>b=lfRS_M?ovog&lPaaeJil_+6h7d01}fCVkjIFi7zf?tUF+n|+3m zW^iE0`K_KvlfjsNCdc%JgE{b0)qN$6AI6Oz*{5%XUeR78e2S^_Wc68m`}Kc~SIa+HN;fjzVC2nr^%hb)?Y;$i_h}akMnAs0+T(0XsE_HX>*g^-7O;A>@ z&DfrA?L_% zS$$-8FI3#<+g3#*76+y%pH{J8qmJ7k6f|f&$j#uFL=3q-@>oqgvzMCVfVH)Rx7 z^Qr1Z#&dIWFqrpWeN>SVa|^xeMeio630(R2@iC*kqT%e@81ZRm1nU(QLtsk!E5=%CF&KGJZPR^`XwTB5td z&&PH2!?%E`Dlucbt#^mZNUzSp>%RPbGT%$vtd>z+-nssJw9cv#T+o&8Iw&rgUh4+= zSgyJ~xw8mFiUQ?-+B!rHh167ru#*QJR6KcLhW9t^k8@nS;q&RFyp;Aa!&6*VNIDO48R5;i;NkVVj zTlqh&N}N=cKHP4`Uwr!eX1lY1`eN zI?7g*1;2~u54b_adGqsSoLD=96k;qTY7F<11qQ9y;xuO9=NC9vRe3b&H1sQcce}Z_O%f5>7 zwW}`$9=pr1DcrHQb2patY2O%imd@*=H_Cj}4ZBZF*Xxj-I|o0c3+>cS4OUI%rw$!( z0(W3OOc@AIs-tEz#MJRPTn`w{(hK9bNPJ2&zJA4~BZE7XJ~vsamBhqrq?>l@N&bbO z`Wq*kd}EE}>B7$?u{-D3GZ;KZRhGAzFG#XR)*JjYXb2h%;=~&YR!B3)KZe?OZ$^KM zckjDcaCii|TE0xuaiFaiiMgl|{D?xC{k@GfEoc zYTnptAFu2LR&3ljF!sr~JkZ0M4*^OE`K8`ZX3AN`MHYNtW711Rk#s8*oWG(KT1h@H z5Z$9F&hwAg(~+?4&|tPbI*UmiTyb;&4GFPN{S)-nSh7o&XU`j4vC$IB@COVhHccFg)M7+jc9Ed@9eri z9CgRG-=(E^Yz!wE(<`&Af(oWnhAqwgEg4C&^@!>Wj{AHaB|ywwNCA?(ZOjV;hbspZ zf)W5S9%n$^{9}9$eYR!K3hT_Gbu?2`Lt3y8OIOKGDrdRnecp5&K;#;ynWu*KZHG23 zd+g7$K)cmeihKFJAt#N^g6ZF9-4im;J_1JSejk_Qva-sbm?&+`)R3tCpr^l*mwVQ? z)pVZ+G{78gqnV@EXED1D8nQYJ$g^tzZL`ybT`|4gfyoZg$7>+Cdlq-xY3NU++4jPN z8>w<{L4fh`16tdd&PDWUgCWaq>QB%TE8%@(j&?Y_j5vL)9h7EZp`EpwL1GjQ@OszqTcArN4L!K2lhT7TZjVh5y2yAvp3F~YEC;u z!AJQ=0T0jf9dT!<8I;u2!~ZMsUPu}{C`E(n#MTZ1pM`Qz_NdS3$_QG0-0?()SBS_& zV^R>6)RdyNoi~7{qPyooKOkfhVWm`sW6YcTlduO*!peFqtUuHZ13xkpw*Fk(ZJ%tj z`}5*k-Rz=MBcol}C#0hmg>3m7Tl8U;ombR{5^Qq4dK?F8AZ@Oz-V@RqUxE!Ls)K*E ze7fkHG=z<7u{=5TA!4ajak^3m#o2HSn0U~ShFkXq^694uMPixQm2Qi0eo%Cyo{o0N1ixwn^|I{tmP=hkZz&!2<3%_Su%#wOog<$1ks>~xHURJY*&y$i5v zAi4BHg00R-oiT_unOnAyYE&2UQt6c!!w4BeU!_kh%;v;_1so^r&D(RdOS9dXG&5&x z%5WcTg=R1d?^bq9C*eM7J)CNc)L5J#xYrbLvS~JB({YtQJv4vnqm&nZMWchguSkc; zp=}+xNl)KI3CZ?$;yL2p1=`Rpvz)Dz+u1bN3dorKtaZwTX+Pe7$W54H#q=_`cPG{Q zxe=39P+KKu&JdHP0G!IAkc0zMBC^d%PgH=x#H*I4yGx>})GX;U1Fge4^Tpv;YwM%A z(>+f4C<(b(%kn0d#!y3UWBH@a2{Q?ngR@V<>xLFx1#z8A9Pfm+R_H$Ce|>6kH#9~P zR^QQQHk*W-dohPtm$rX=fj@VCh%3MMnqVEj*Wbdd{TuO|#)55)N4;`uhYw-F*`dLa zf%!+72E+V^4N%nDwUBF{c*rg7@l~w7k74)bmj;TlH#J(qR-5)+`PDdNvW!6^G-{BX z&8#s>&!9d2#|IcRZ1J=PFhy+~Y=bd>k;pbi$)`OkCX*y7t*Vl*{ ztVLyG!GQ#wViVTcx#{nL^#>S<6W9iGd-j1p>PS!vU`G<~R^`a?XUb}pu4#hui1NtL zc-;4$x!FeHfr6K`YggBP;rSC$aZjW^)JmTF8()(cOQ zPb48=b9nm5&AlYyD4oK(Urivel!`c}#F+BOi#ViD_J3ghS}QZ6S6;pNA(Lhf)w0Bs zu;S4#8kPg2CZ=I5uLd~37~jeA+cAB|5wBm_i%6$nmUY?u&yXJHiqF2VPi zK@3*T=bWaOO5rimk+_e}dji$Y)A5sy6qK@*1qVDJ?CdJ#A4>(2a17m}N;++MV&Oc4VOm?4C`o;;*HgiIuC0<0 zA)#fTC~v31BPx)DJ0R0^B}dEQZAA1^?-k!|W$}>e-XT5*@f_ueN}GjgBg>nk8Z+>8r_eM z&V=NeWuI&xV!Q~A^EUj&(>+YWesfr=qo7a{YVi^Nkw`zn?G2e1WTDd)KK@$~yFb&} zX=cyemiRV`@trz=@g)m=88ZBNjbpom)mazi|cjeG!iQQBv)8ABGClrTj_` zrYJY`RWekCOwQ{REgqmqfOZd%KnD)(YQ!Fv+8-Sh9+X6E@LNC68R{oNEkhQy4DW-} zqckIuoQJCS?SGFI-j4e!mis{R_XaR);!7A0qtkkvswvuabj2s_)yKFxobt!&Z!PRZT<%Eu_@9~bhTfPzO%-%_7=66AT}v8uW9zs$ zkn$Y!Ya?G-e*&VG)cU4rvD~x=;L#8V35Or{U=!kIcEJAWO$vpg$*-8^+iGWl)_l|^ zG36Z?dQudg(>Ug>gbitsLOOl!%^&3WKmsDUMrk6S8&SK=xg#o*L=&4P<*M-{OqkGM z&dfjYTRGCTY>a%5?TeXSGT=@Rx^@BEy?ho}I*Hkv`ZS)?wTQspja=YUe2$<-Sda{3 zP%UwfbwmOz|IrS|YbVXSYPEBS*nSX1pTtP>ApTjmVlY9;YJ5Ac zJz(|c$Z2{oTH~=EMdVe6^uSs9I)xA|@pl8&(+*$vJLYWtjvXtm|-eF%?+Lt=R3 z1*Ymmp&Og>%n(V)bw>{PfDuEN)gBd{ASDsWton4$YM`Cydea%60vb2{GT%L36}c>B z=Xc6AiCwtHIcp1rkS9|Ti@$=;ArZIL3#$fd;l!E~ikU=$%GNq#6r~eT@B+csr#X`} zfzKoeHhqk^r+Mm}#h;-?zMM^drQ|4ezxO%9iaUzWHK+}qUU&57RVh}i9t09byBxg=o^-kiFWt8HPVbFz zd8fOqSB@l^@%RM&)HsqZ6=@ka`OtZ4dB0iH`W5s>8Y~;>aysgC4h`J0 z;3k*yZ<%Tq!ZLr~f!5Slssi9K9xaoC+a$qJL9t(7`A4!)aQ$NvX$Y2x4j|8Ke4{n z#H!d;=QF|$W`jsLSUn~Icjmtnl^l!y#>i;0bUX7_NB>$ozo``G0LOh#pdVoK=8B&-EH=!6tf!|V;3hp-$#w zVs$lo_%bNqaHy%oQ(L0@N z2)h~@-P)8t>^5GV>E9kwJ&!XO{8sDaWb<%ApKq1lj<<MwfE+AhCu^J}R|wpzV0Q z8CyLWhsus_y3FicAluFT*Q6cEU7oq1&=y`I+fK!D;k6O*J0~8=HuI~=N5jK$7wIxm3yb zqT>M~d>Ac!<1JKKRHb_XeKIGKey6gd^py`s4OfjwMDdkOY{>x3vTsp{!#IpxhoYrE zK8gyJ39y0@Bl~O}NRGiI+dFb+jKY>^p195!P2$JT0*Bm5x5VAZJAB3Dxfjk9p8k5E z#IA2wUrow>Uc6A+FNX*NpZvB=fR*nwKo*R|z2J`?a}X23)uBAYH92eF*eaB3g4&zN zj!$>432eybXzD6QsUut*jJu+&=-W6FQ{t~NWsRh^VaLNd?{a^use@yR3u>!CQ(4SE z*6>=1+PH80EVQ)S*kaLyLfBCM4?99A&ekFxH;E=3W`ko;;-E4rhpq?7;TQ;-Yn=`W z@?qZweWwaYR^lak*@k*yT4`FWxF;?OoG3 zfoPBuN<@Al4tMd4WQJHVp;j8)UW?ql>2XLPD|&ae1Lt_GQcOi#p5hpd3N(xv^$n$* zB5wR!McOFR=gc&-F^`Ekcu}892uGJZp?{JMWHI*Aw>4A93vhiakWp|{qYvBLv{Q$8ii$M$Q$Ca`vHj+GIPHQqgk0% zMw<=_GI4ZX1}+>|#+&>&m5qa4FFQCGvy;@V=egFe?LN4ubPxv9yBtVQqIleR%$9RJ zHGDoCHVT#^tuku0WI8+*8iTn2V~&SPb~{g~xLadB~vx8yE*JgofI1uF&n z2Y07VO7ttJv1%~GA6XiKvFfSQ2KAu@FoesQA&>({9%}Hfm(6QL=-ihdk`n?K!eac* z09eF3wL%d%A!vapSIo6UflmA-7}#rZ1>lRaDVP``??^*f9rGYL^BLE|nG=0!vYT+? z!(a)s@(K3538cz~#5l=F`#ulsJ99M=Il zef)?M0+T`->|0^~{5~GG&ARkWLsS4;(Uu9qlm%4|xCO@v3B;)-{#L0g&i;2X%2|iU zfnI+T9-S?8Ls0><4?5Dsd3={P)yO0|ae?U-KhmUS0F$Q3D+`0ps$~6mIQ0#^#%xv# z(&n`P3+oY%K47LcL^q5Uh=I08f+3Tp81gsse}4m3w2#iJ>Dzg0ud)Gd*yS#urY4&; z0Q4>ssdqgCdMcxDoa3dKh-lEe_pLc4grb^`4Q^D#uiKR#hr)D$<^WF0^+a=NhA}ya zMddIXb&1F_^)nG+{`vkevBs#2AoATFF}Od0`udIY-e3pdyS+cp!f@kX1h!dkkTX{|$TF%!jK-IDZ9BhcyVqISUWYK5%P}Ta7em7mg2`Bh2Z(vC zC|@Xi22#KQe+v!M0|EhKgLt1Sby)pxe-cz}^5Ma+z#0*5^>{n3h7wO7AD{Xh;x}D2 zZnA){JMpGWnwWpLSn`aaOP&}f0qpVkE0V?Ojwwc9RR+W%4;;AR77N!fyQ-1D!(rKj!T<6nMXvDJ>1jTuU47BJ{@%1EWI#F3YeV;FA*kRur5ZcyR@3I9d-ewYw9Y1ju7Ci$U z`un}FDK_C)Uit&;8x?`=$l>|W)K-YXV)8?NWWBsxE(KHf)5l0JvZaxR%#^Uh2+`?HFLnxog%x7KL{^LTT*lJN_*;+++ZjC+$amdR{3*R#?P0qZ_ z#^q&o9R$Eq_mDwiAw^j6`Pz@l36P1>Drd!y6YjqZWn0t`En}O`x68lGGhc9xuCy?# z(6)Wysrn(ph74#cIYB@R%&K*EhBFE9DPS2c7ZGj5Cb=`9)D8iv{er58O=&+&dUH3O z9T*ptx6_O`cU%+fmnCIoa%?(!bRX1<9qxVC5FEF!e6ZR0yPYYVC-!vxWV5*gJV6l+ z$|}jet4r{TO!>Of`K*-(=wvm3;xe)AR?=owyRpz5px zR2IU)YXjFV&Nc(KLCM~9cAR?4^rz>7pbqgNXn)^s>a+gRxt$ujwCR|uq;@te`)oc= zBPu7+2ruBM+e~1{;34Z-uf(S0gx~LUpStgC_*q!E+N4SNLif%VT^Gr%FC9()gb34;xs7vX%|%89#tyQYapa z+WmfZg)bz@)SEM=5){u_WcnQ_t@+Qqq$~qPaTcIpGvJp$s4JZXHEW-IHXDP%!yM#* z9@58q%NR)?*9-U^%r8jQ#n4Yb4cB{YzE}cPo8c<_jbw9?qV&iPpVZkwBpj*```VeHWUEBO*m+1m zh?!fzoKnk6rPQi$=@Eq}x=>JMNeLcF@Q9@{zxzV`0Mwu=g$ca*#M1Q%n<1h{Wc(_4 z-@!O``a#7KV3~TX3=nXFD2a_)mOHKUJzsD3MSzM$=ve;HhI7HE^3!lnd`nL@xZ;&vZJf!7l(=?_JB%w3)z@b&u5ro-LLi`Q-saxmkV-@kUjtC{p(omaR?iqSwdk}PXJYYyIVQ6UJo zqSa$?Q|lrCY$VMb2xT zVYPDww^5o&8dewWn3H41(G~HTFW?3U4D+9BW_Zqq#xX3aJ}#?!*dsiwdj55VOfO9U z-ZE@&kkWrNyDAJ`w$aeW-vL@0we7(sZLs|b+r4WH$a|c~@2P-*SUMXw!5G0$P}0EC z>@5YUINH-0a73Ugph-YV=&Kt_c`@$N&6~vX&_HQsXD;yIM^%w#T31GN#k~aJLNr#5P-5CP6|U-td~O)EBuIZ&&164 zG2&Q#Szl0+k`_fxOZ)Z&OD7vfei~Nu9PA!Vfz6y0tG<}aN8rWLH6DWPHa^|=tzlII-vjwoFyA4R!`U1ak z_`CTcy%hSaDc!WQGYiGAPacHstPpMP7F*AJP_V z%IJqoHERf)s%WW5*?lpCb%_9rgFAN+vge}oue(^u8m0#%DGUs5a^Hnfq8tmgzR;rva>bg5mHs(I`kt6cDw zZO>$`j_DK*Ux!7|m_U8LzxUDlh9|v>2RVo8yufygE%8d9^7W?FIc@lWi5c75FR8d( zHFK+Xd>!)qvGB}@#lhQ0aTvaaTA#z1RuBcO^elx?iz7_7?BFSGe|+?D$|k z?eyfH#_fa#?-ni!Jp4ny%e zZtefLC+K^)m3{mi`(g*}eiBsK!|zC#LGPffDp3_YwUuFcen>;8GMv(2VG&kJvhAA) z{MgM~67M%Cfu0q$@9dRFJOp}%!!6GxFA`Q77Bi}U@3EakJ?6n}WafBq{oP0b_`6c+ z5)ZQpO6>*zFjY3|PK(H~mtj#_Ko8KoxsBKr6E{1hZ9c!z$r2Z_Ey`wUK>n#!_Fa>d zx2q)ERT@8rscR<_hK6vx{Qx44YGnV;{renm#%MBxKT| zH!KBgixMZ;#nA0{MRw!DSU{4Hz=@>&rgD%)A<`q(n?0~H8BZ_P#tgG!;f2*86LCJY z$e=tT^rpnNE~bpt7G>t9n$Nb(da1u?;$R6wH95tJ6$>AjUIAeAi@i{F&37-u|H%3z z(A)Gp^z5m-$Mhhq;|i>sP-G^BqMY+rro z4oenQN@Om`l==T7Q+8s{8d>R8#*Mcx+QZWjF-i$}&y$f;O@+8R!D5HULOV_255-8d4YaJdigmm%cute^Z# z+KkMB-|S@yF0y>s-DM(gK!H=3@0HoYVFJ*Csv1=K8d0f8Oq|CEV1Yd)RNC_M1`^~c zVAX&lf{rU_@ z3iz$-0^|k{g@@GV9cle9GRprI$Ow)e!m4J(1jXK7rw^ro4_xArG8m?;#xf82{bUib zfeT|b@eZBYilY`fEDn*ZkTwAN0n9R-DI2P+m5tvK`iU5STd@i!J`wflHJUAOutnno z;K4*xN^7seCf7yrj>*Q3hnOP@^^FPHo5%loYNqw8g&IJlSA%nfh~Z9`N%jm;k&X8M z$7{F_puqnfK4t3OVrMX2VR0^3CQ$K%HZG9~Xh65ytyS=qbwryO2x1YXPWL z1qnE9V`l^D=#RTlS#vjVfQhf06+cn=(782MF(pm7O$_r1cCVqC>qTulV1!yvE)$WySw)&7e6&>dEgkN4gKhhF(F0W9W= z8}GeFD~>o&kyOs2Dxv?0xH_X;g(Lf)2Z1+Kpm@kbtf>2cegkd~G)ZMa!;m;_qNsq% zC_!$1D2`6*B?vN>ZTN4iEhb6>%8?#4ND-V4yuD;gFxJ$+36l0|kYx%EQFDkaHMh>r z%kgml1@pQTiQNim{#(dUsJGj{lNfGr36Ph6MS1h`t)y!JGem)&HM9MSC4lDxB2?Wd zNGI!+BA~9qUl7mzwX!fNI!JKZv)t??gyiQ^JW@20L=4XgQjQLP8Jq89!20M%PY1f99<@YRO+1$84{u7W zsktF9GS?j`R)3hUlEyTgz4lnOKE>yWs{f#>WO)Q_&o6RqGZp!;;_8$kBl@`K8#2Od z>9>A}9)%s6d{?EbKz@;sQ%b^N_!x7nMmOs5N7&Q_P4ZC$%~n$*4R+YE+pop9X)5|E z-xt_c2XgrWekoL63|ot5br@{;r&vYsNaLj8Diw!;Vb;b!=++pugMhKGtIx6{8I)*` z<*0c`tq)A)s3ml4MN(f2s5PRK`Waz*eZSbne!Zbnf8lAak1W;ikazTGA}NF;8vrge z_{p3aDuWtv;v-m?*MNOY5fxqe&AK=SbgW7IqVKe|R)rMs!7`?Ap|DNYz~N@-d8w$A zUFX@2_S*i0&w8DDyhgxV`{HmoX1f+~y^BioU029;)xJW^@eDQ&()IoY(epA7X0k+K2j=W4(c!m8=8xc5$FW;MgX4x?vlCCvanYX!l z<*S&|9Y)~@oNS_zC4%X%9k!9)YF8WT*aiA)7g{MEPULw*Ua~I4MVYAw#1A=>L6(TPdrT@3_|MNg^N*5gHNnpiyNTVe-l(`!$mBAC=$XCcQ(xRVJiS59({6sPc zYaJ`?Z``%HW<2pL%8uM#SILWmTgBUrMKdvWN4`?;Wo>`a2fc(C5pd6eZvO+LW};dX z74lG=g8qKlS3rB_ipKLRug}EFmQNAFX%tuQeGY|*LdDrNNA`XU!khU_r0mPvA`93x zb+k>JgbgqwyS24TpkbM!ddpJU?AdBk(q>xxJ`k(X`EcYt4~xydMP_gcdhL3T2jAAt z62giWGJYV~@g7|kkpg=M#gmY3x>NYPu1>NsN7}J z0CO<9lf}cj0Q{Gi!diJXF}cIe(NUQ9C@Dubn;8=NBlF1=xptjGfh8OL0~+O3%;Vpf zSjBc>YAte%H{ly39AxgxSP1tkRutp01cGgmLV@Mu1+hCqgx{u&<^GPW2u8k%8adRP zTn0%G)P0SxZM#Y>08v9${6w~n%f@EPC1v%Ja9+qV^1|<$7~X2bEBIELJS$JLI^qDN zA{G1BW-$|x3J@}%O0pb`jHQH$9G#xZHilh)br3<*b-Cbpi_H*4Jj=!dhzuz3-^5vk_<& z7^i2KlB$l;K_YrTB63tOA->)-aI7lqk8JF|N`7c(%``T@yGy*!6&2Wh(-sFwj>v!9 zu5yl|=~4vew#{jQDj8TII4$4a|#x8zF!jzJMFd(^evggeHM;HQwox-x~3aM{m&50j}!8+2u zk2oMo*AdV;v?uHMsuBEtt=f(pJqi1m2&GlKuM8gAG1L5| z9L!MU$RY$BPLM4PKVoj14tYL>o!Eqv?87ykaQmyKQW=hr+%$s>dY=YuFil zCB-CA>bo#&-BvHSVx|AMe5@eWMW0Z!UXR#GDNFN%UQ#?ac)$L7dHYbW5K?;?R&;s( zs2R8o!8Hm46wVgjc~RMhEv;z#CK7_q?hvSne!Q-1U$2ORtH4ojSyZ;<1Q z{d0V^tQVe+Iz`UR8fJNT%nmyX%fhi%a?{ajO`Ubce$#c&U2$_aKnqpV)LKL{3{S2V{JB{${R1^e19UNkY%_o6~Rn7Y?5(eUD>9*CL|YAub!iMEKsr;NLLii$f0F`tTnE zFS@?7h@ka4xvNWM6UYCgFQ$PB=S5v7rOJdRBnO9IgpL`UFKkjxCz-)Deq-lyjt;9{ zeXsWxgMI5KKcck**!3H9Pku%Y^g&m&!-w$zvEvL{QO~V6-{dx z5#0~e*?JqT8vi9M@6u>0MgXIUi!_?S-qne)Jd~-g!Z_PAf^Y~1BvG9xl_ERz( zJI{dwdY2dFsAMc6iqn-I$MyAz{@}g}x$bxwb>Mz(P8)c;7WHM=BY9@Y+WhF2feDi) z8^qTynEE`&8bUu>E}5DHC~*B+8j=D4*rTO^6J1gucErMt8Sno{fgLo%b4JexqUPr| zsdwM>BCmSTGj=mT@9jaGc>1q@LGq7HT*O#cJHzT=P(%oFJrSkstmSP(@?=P5r(Ge( zf)j~e8kH9_Q))I^j}+$LhZ~0V+O^KWqRUa^ej|aL#tW)^@7|-{xX}?=6i0_z>I%9o z1Igc|zWN`Qigf>fZ_iQ@7-v&hxwkG!V|Y@{(#2ssS<__^1_{v-vB4T7Fqm1Y7NhKx z=cuzfUT3j3CPBp>N#R25JU>OyW9$!w!;8~7krd|t!Oa7C%7}A8r^3KeXU%~sdsG1k>iNTT*iFs1TjE~9^Qv0*;Ls> z<)sq2fVJ4d$_9jFMaxhT z+|5SRpFrPe7mxsI=eNtDs68~Qe?o-&LJvjlr2k57WB@~wm}xzf`!%X|wHR?E6X<$D zIO0(LcjRz}z>?gL>j83(Tqi z8`B5GI1Ycji-CWJVcHFe*`Z+!eJR5G<-}$C&r;O%v59e>A^Mo%+tHZq)U<~SWsEw? zx^KTaGY{5=AVb_f@e^AHroW_)9oc05S3>_YGBQv%eu0$2{d3HT9WJN2M8vTxMI!qMU?dL6yGPjdKejf$$qg=kB4< zL&S(My?P14Nc7MCP$_A&WZP#vw~fDxAPb&8CWBPv<7!c%$(O%!X#7t(l!RRWtCf8n zH|=yDDXdB0c&^MD?Yw_ddP#8^^TeBhznicoCux(PR+#@-(T+jd6t{;J!0rAQir)h+ z40cKL8DbpZD1$(}pzJA?VfgNbgvDj1Y5J4&*KMmu-v{r2X1QeuFGP4Idfz4%kARwh zEhik3^;NK97YdZ;w3-$F`=yvDAHUX(0lC%b>D@-qjupfVJbjkp7_5@gec4(m*PQF_ z!8MvOq$UZI9p9=xDlEr=%9(dwfrEsO{wJ=1ZF`oEU7l*noV;G$e+c&*5}f?(hOkb! zN#IEp+rN&F^j-g=JX<&*;X=g>|FQOocW~lnbY=o zdtnODfXUud*lCU%fuDv!+e;g7i=V*hI`KtW6shtVPY(vAbRgFgF8zWa@C#6+U!Yte zr-#1+!6x!1%!{R^6d^8PU@K3|2%x#YQvQy6!Z00X4eu6#)~u7p+bU<>*OVdpY`vkY zO@BTX8{`PR%Z0R54!z@bniIPaGv)lBV=r-re#2 zm_1*@w2zOw=1=Q18Hf0IfxT9fJOjHG$_G{)#`ynu#e5!uTU$Pe%6iBZ^|U{$zS=-5 zc0@d#Js60Qh^t&3)CCmYGKV$}>3oq|$pwFPX#ivYX#k^NHX*l z0XZxOPJvYeOgcw({0weWwS3ooIXKxUJ#({DA7zg~HKBc9wD_rA}F^}X9 zte=E#jL;Z7?A_`?ipjqz*vKsE#nM~~l82@jBxRk#)iR=9hq-*j9Nh36eh5}Gp=1Sf z*`Df*G18MIIR1xG&HE$QegEQt|60DkI8fiK^O;-4)$+h_pG;xtHC;XICP5h58OFu> z5|=S=z26O_l)}guCqKN0E7fTNW_U{RpBC`nWuj{)#P6KIzM06W6YCEX5j@stn|}BW zjiLgpmY?zlbC`guG+b`C1PN1RZ_`+?@+BuhQiHTpf&a9ENbD_s%U7}zxoW~D^he^? zE`y$%X(a?qL|5UQ>d>GIsIMSVa)mY0x{{1uui)Y&3~*4A5&dICM6A<|6ON9)w^=ys zeALuVyFqkmw~5stFMG!)O?@KY+k!A{yMq`3?LaU5N-$CBF!)IQpj|erqaRV)j4sW9 z6(Qk%n3(Wyt0e#uYHDgkR#QY6z&+fZjEfs*`G}E+=nQLi>p`*n5D}<96}qY?oq#{# zbcc*X`xj$dP(4V=hg3+HBx-%+8uL++N%TJ>lmBX5N3S^-2S7X;(0wZy1lDC!RP`Ss zz5M$r^zZ3-s<+X%M$zx&#TUqq&dC?RuhF!kDE_r!EJ*ba#U)t*#mUi;VP=r`OBrOC ziAp194r`OxtUOwuZ=kFDQq?}BHIKa2K$I7UQx%U`HG?$vp+Cx`^K>muESQz_9GwfQ zd=rxne+zP&o250^N&hTeBfrJZa-AD~(NqGIh)}DP=syE1=%(Y{;0lS^L4Gu2Q|#BZ zTmnv`Y_PmSoT?DSJq8RvY;?a|jEvD7Ib}<_8f4D&5oN3Xb$mb?ce&|TKJYovT;~5Y z1GgF4?m7O5FZ%R~S}NL`5~goV5W)tGA~(*mwXeWPRx5@^d`<+}j-cp&q$RSDH-Fp@ zDlIAT!JTIRPz{_A^U9z%D#|AjeXIEQSCZ2BDPfgN)@d5y*-0w&12R~|++$?nm9{Mu;)ClFR5qZ0{C}7@OiJa#8 z3ONM+4F#+`r41q|J0poc()R>srye_x^MTiw`X-Tk9cr}vDjhA3UG?qjMvAf}lF^e7 zZxxUQz>ejk1+MSzTbr)mW&IN=W{?=$uZOi{v!i`iLn5vGTBUlV$cPaYy%aWz2X8-g zYBhO+*~2Z?SYnwtk9K88cV@)=Y>-IJw5CLV{vQ_lIsvmlH^6Q z>D^8E3^pV#(rg-`c9k69PQub6#!nsNSHBjT0K;~fTHLb;FE-H{IU$%X>Oe*VtSf`I ze;5HEp~BnsonQm9O--MNJ!_UR?l(P%gZ(gp;;|rFaY;EURk0c>J>VFkw{)&Le>-txa1b@*K|P#TBgHHD}9FdICt z9*0aZt0}H-asr`dOd;kJCj&73`^2q}3wtA7(*3Hh#K0b3=5k6%a3XX`3iJ5X1cZ*J zZ%vC?h-UwwA9~*I1P?(m9x9zR8z)V+>!L#_zQ7@afx*2;@DTmnU}QX3C3d1&RK>3jij{0249&=R{sl)#uu0L+XEGot`O8e-(x| zCVs_@Y7GJ9<;UtPPt(w8`>Z7mqGlh4(()++$Z1LbBd`@Olo_iop6OB?{ zZIOCLf9e)gZma(DP)|_IB7I$56oc}1PF%0dzVNcJIFfwYH_@Xv4EV&6*!vDZ99`cH zJTeU+jQN|-E7Ou08U(M_nsYd{ zo0}hpqU%{|h^Y^Y&l8J=RhDfl0xj~!a-oubq9D2bU^*zt_KL-!#w`vGVo*7<2xVhv zKKS;r-xmSz&LvFpm{4RWf6AL$u4}bWoF*E{1L-09Vx%|eRrF|9{-$?Q{uBBLM?ZXn z-&~DNCb?coz$S(uR`k(4nsFy8$jj89uxf%#K{spuPWcsTuL;D;+tti=AstZ11A|83 z^4ww;@O;044JuQq(!N)Dni=crO$vGA62Te&ulBw&D$2HN8v!MxLmHGA5u~M4MUd|9 zmhKotV5FpFXh|idLpr4g34<1;K|mA+Q3es-a|Z9{xo_Y5-}moZFN?(j7M!`SeV*6e z$8l`@=9E-!82mw5kd5{>{TzbFe1 zg3ay4oVDKX$?+-Lt|4yzz`Cf0$5<~2k~`fuDA)7z8sIeoPa$apf-) zVOKeeCpN!vicOVsJRn--4>nI|K0M!@%u2)S_`C zLwXgMK70w;RdJ0Kx*IA?2~s}mtWuh!$|1&#q~-I80L}mrk_M1PC#t89GX{y{dWywr z{nimBKg*GQ9<3l*uS!ruVK=9Pl^ZSiS&l>VZueD%vWoc$H4al;LbgR?{VH7L@7MOw zGUUCODtiM_ds|WT547l&rJY`1$peA=39P$gKkeNP0uI z!TeB}3HRL3nXRMCuL8)6FVnB#&ru97^gyM?=aWwAAF{}DrF z@ug920A#q4C*_5DGwBpHc?b+@6W4Ugi%7J_hoa$oZOXU9~o2wSth7CswPnFuLnieF`+)48 zvVbojGxA@>7)NA_0F3xC9@_$nfGA>u95zgOW*TIcivLB)-<(tf?oJOh-uV#u6vw5j&*p@hnM3o526M~tr#fFnB{YC2 zEcBIe0+1;f9tI#symJOj6Y2oLB#et$ntSmPl?=k93Z47!B+3<8pffdAhOP~h9y3-^ zR$;^<_;+|Vm*f^#5@iQKwWLIVf2V$@CYp0QasA1>&pWk4P#dTQupS@}zt7l^9s}}6 zGhmiHp=7-4jv3m$2E<=(9kczZVXfd+1WwGe6^>Iw+-b+`R|_ZS6|hG3XM^GwV%2W=EuD&e&T-#`aZPz$!-y3~3 z_tCHK?@1s0{H{jE9CEyeCKT0zJ%Z?QaaYt}pZr`nT?RCa?!fKJs8j=UIzWse>O47e zt^gd78OM?weo+0Xco|LSsRhNxmceHCfayiwp)sQZ@MltWiB0x)KmjuJ?57s6><*RX z;syx*&ceBedNkf60*Y@NHS%N-Vn%k-r^owZD{(@*fVg0@^}J+2-^4XSO=Olm$+w5# zD#&LAznrm2Bp!gD{_HyK1e_kpg?29OB@$ihRYVu^zf#530A>B;C36QuY+3 zag1iVT+Iu5h-MFg+H6kR(LJ6R9!|?fu8PZI$SL@PC&wXkBG!G6Z_N}(4)Eo|F(jLj|FvTN)Q@3U@3H^UtG>FMe~VYw-; zM7prC(?W}FGAZ_Z_b%y^lOJ6+d;K&`jII2$yw2j)%~}~+YDB9T4@~Kl={H$bdd*yC z37o|=)fhj8a$@d$Aw`QC)8O=x7%+xhY^JLp<|=2G-Rc6M_fxxF(3RFm{d7U1C}C`u)`m`twK}4TwPs1Ep&x=C0xj}8lZQ+{<)@?1VQ=} z)Y#)gS;=}VcC*7?3S{P6VQ&E5(cRp1cferb5FxTC$KvT>px18m|pL?Mx#$H-Ugj<+PAPlFB5~)<@NyQ405U>eFCUI=puq@ z@XM-xyLmi~GptKZfGx$2pn?RCN&hfKGo7Mc!|+JeU6!|v(M?)&i^B)W6&aJ*^)o!4 zj)9_o#@0fVMu0c>09i%?t?u|fhcNYUYeuzA89Q?z`Ow^;o4th^bbP;$r^G*wr+4i@ zbZh<5=!H+FQ};y39!u5uT%Xm&?uYx));+K;1V=ILrG)3;PKVT6oJ`1 zqi-=!T``9YRd9|CPu`tO&F1Z@Hh9P(POj_`D(s08j!cM4``$q551Y3z#3gf3P)!EZ zG3+dSdu7M^ZP+Aj#n@s$%ND3v=M1M>>@T|nQTo#P^GhLXq#k*?nV*ri(o1jAjAJxG zD(-{ddmg3y=3(nK1aJkJLpec6L~?D$yk-A+N`eDiLhk#S@=N`z#J+8-a22@7L?(lS zWA!M%Q7=~xct{z*_NubR9@Da!QE;_^o!@`teU33+k-xK%`CKvz)#lBW`oFP zkVyiT6y^J&Js2Ld94K$N3oM(1;2Olzu)miuIam5VBvqs$;7U0tkB2{Pe&1CvxWFL5 zaMSDb*N?0Fw?qb+{7tQ@sftjBQ(bMCBV7N(^*O*yzS$s`2^F_iC7}8kM|*6jjEJS% zU`vz|Y|-NX6bu|GE#Q2>vo5OqFvK2G6)ZOh*tf2+xc4+d+aXC5%f61z6x9U?lAdra z2UJjpX_6?X{S&Cb%IJsl_BBQWgKtN{#&f^nD8GIo@C*MgWFQM$XakSYLyqDwGqpsm zeb3bYgRhcEj^q`_MA6*@)#m+Lz%TW4b_Qf?ua%W}6!2dveujqMhhm852D5&R%bsTQ zulHx+`!fXeT19h`j!K|blaoh+K78pqUgJZImS`EV9oP+=0=A=)n|(Ih7fVh34H&<5 z;p_(T=)3`GM0gvFEzdE)NVtTncpl2bbYlyIOszdLD-8}CDN07)drrsQ8L_`hS5E-~I zQh>Fhq<7w;X)~ynD{Q>b$hNli;kZ?heWise)vhnCzRj*4hs;$Zb=Fb8gP=_MzIawN z6XfDnpr6ClJPTA1LKz+?7f#Wk zxNzsB#sND_wiKE{8OAKMaedF`@oJ(A#A1tX`wNxBV@Xh75x#ofpFBez3dm*r;FP>7 zJDgdxe!qn1L`PlRj)FFm%o4$Yvwfb%^f9~e;fieeb^GT=fkUBE`y)bE2}n9cUontS zDlq2Jc|q{e^p;ioZb9`ggu9Se49r0D!}wX~*IGfk9*s&*n$V&o;Es&7mxq2RN#?Pu zUr+@)=b?7dCT=Ypk)$lSpZaVGS-6@nc89AJ)F}wAsHL?*REr7x;eVi|)`4zt3gNGx7zxzQKqCFhYg1N}j z9b#^XWM#j&xG`qrT>j3V=VxEM<_av;9DOrl;m)cXcGcY(B*c@UK}HFQK0hSX;HKmZ zQdlu!4LVtJmTd=A&>i-qZil?<1XME4LgYPV1TwOF&w{{?qTr`pCqX6-h!BTc>EjQA zAS?ShuS)J;OGJ>$;7pCDh=VgF+2KA!)9}MfgP?#f1bIc!B;mXVsO;e~# z5|ZPn@qirAB~$eH3sr1^%{~XaEyRYQjKy1#F*fcvu?wO!nfXih9C?Z;CyHc|KQ%{VkAf+G|OQYy-;8+O2pD$i-LBKp?G0d8%=l@(7z69}l7iH7}&GOsz1 z*}|;=r&(EWhbZ(oX(Ud}o&EEm{4METGt!x3tMOqxKwA2BGgVXtLJ$1uYdZp#1519p z16r_NA2!~0L&RJ`RhHT8V zZ^1GN*z{fa#k+rqZR@dpm_;yLqNHrUvbP+cp$a^2h2RqL^OgG@s`vIH2eJq$RdVW) zs{<8M3)g-vhLOTsmWWe!TC#4xzsK$G93X&_hqgbjWpOW)iUV3R&GhAFTPywtgrx6x zcX`d*-z2O3D)hPCBuu`8dDDYjSiL?mI5oq(1$@SRJ`1}wE^dEfKJaR#=XZSvX-Q>L zu{Mu{Mx5@_C66=es0!R!$0f{j%-|FQZwtw4bz=r;g?Qz64yAfoz6EtIMN+KQ!5@wn z1HaG{nYU!eA;|4uh!}NZ(D&PIsnEiXX!VwIv-R4KRo%i{L!urlg&)!JGVR!U#UKcP z@1Tp{=-bUE*N*^CJ{Rd+@380alF()DdT9RjJOjkzQ}YnwsMGk=SI6!_11r+7)Xfy7 z3!nzEnt6-$d2!OYwR#TU;>rU;Dz|PyqaHO02z==AY0XjA_;=f$;meSW!_On~${pB6 zCnLi#4sMHnAA^9Vk!g|^L5kPtH0|8FCr*JT@D*1Iwt_AYhOw4YW?0Sy?T;oej*fV2 zg!aE{gcjH=F6)t&61_Ok>1eBKGZ*gZ9MP2pY&JyOV=2y0lHq3C(@k(6ml>WWUQhC62CFQAnU zliOc6>!gj4K;jJ0FfYF1LG?yRGPQ3Rog5tkv!6#S630bV2%PJUB1;PweBRG+)wO&9 z>socL)bgm;nD~aOy5GjOY(?PG&(a{f^59S-1A>$UWB|@;eCF}rA9VWnw>!`WMOB^G z;v?!0Y-t+v9_h@AucT=EFFA|X&mUS*6LR})SFhMs$}Mq!3KCiMW-E8fQp(>&B?j`- zb=TiL%)Lm( zINtft_Jh_BIhzrUFY~jOHFLbF!rBTuV%*=hZzF)2@m2NqZcCO3k}mf&Z*r=FbEBU9 z{XVqa&HtAub{-8X3pPGEkvEQPPcskPTos2qG)z!(s&1(YPwFCsAg&t&oTOEi27Wp$ z`tu$*X_E`bYFZlk{4aYqbhd^K0HpAmJ7dLQ=Tn?uzD+^FB7Hvlpa!8^y1T=`0l0ve z!>xh$;;gW~l&$4evdHlP?uJA>-Y6MYzXp3s8Hx`D{uK4@o17}g`CmoZ^^ zDf|0w*+TZUt`kD7!8*mpCE^Il^A+Y+Mpw$sOnGJS`B68a?OS;FC%H8PglwOMSFi6{ z-ytp!3Fx)Aj?xo9&>2Df2waX9wfdL=0Nu^WFS9tH{Cn~u?VFkwxoo!+n79=TJ9@L& z-PPNZkW9J|c|#;(I=a;1*P=-JZJ?>;xfg?cu@{{rATv>3(f)nT_LnyL_-~7~mhmi* z(h^`%+qcyHZkR=i7|=d+=^vhXF)C%iM7y6sR_bow^4)nNfq)=bAP#}sX`SP@7euCS zzKe>Z?m386B7s~1LTb=Zv-p1RABJl$H&h|+TLe^%CAwDKj|$iC^P#V6KwqN>v$XwtOsj=9f#D%uCXO|ZwxN? zJj5-*p`hkQ?S5(zc7!DUMA*4Zj)(19VB?T7Co|)$v7)+Lz20{-P`Tip4UUV=Eqa*1 z?Jt{Q`zc#)Nexzg%D??l;M_|haSO?3*E<>JP|Hw3RqfrcBSShC5LPhg&d9zG6dxx7 z!fc8N7pm>BBNmz*9)_Rhh-%~lAU5i2g`AbI(nj4t;C+4K4iH>GNY(@~mDrpoq~r!1 zS6z6%mE_01=QKN(tPv6dH~al<94&wo@kT)0hjV0YDsYuaVAn2_%Lu$ZxOw@qkfKh} zoBJp(ZWd|SElV!$Y9>=e`4)JaM~8&)g_#XEQ}GJw293EJgCMIdu0h4oOV++OLhEgAPVv*kQD(-pJEx>%5p>5_@gLL0)O9Gb?`IO1`6>1+dOa#!Y# ztaO+x1`%+%OqV(YJb#R`*GTV;Do`pWW&Sd}N|1bU#CTtrHjTKBkj)u8A{u{gFdT~y zaz8`Ua`H|GeUUeKb`O;)7e-fGSPzKAldq@-?p`+QbbE)ghCfsy6140Ii_Skf*BBHL zwf5*((n8kSx->Z7?bkbRbV*@WaD89UkgunYs%8<0?kV>Mx7#IRH~|YY)_}hy^xaN| zoYk$EuLs(qNSkNjZS<9l7V`S!5@#H!_vv32lA?lnH7dh=Nf`s%bn6+ z8IN(hrNAmN!6>^LS$8II2(t~-7S$o!JGeujm>c|R^@ACTeRLSnDmlrIRa5$}NPFvl z=Kx}yAjnlj@sf-5#Beczk(+urSQ=bC5u2M*8sT_jQ+&)*!}YfV5{N7!JX`2FG)<3$ z-xt5R=`uPTSK0M@7&(6pLjw%sV@C1t@cy^DAUjl*U0!oi_4<;^EA(qbxP~Hqj9df$ zS)jEX!3BG}MK@%1+|KxBP0@g4D@_@<>thd2i76Lk*pim?-C8=Wq43+i%RJ14ueeA$ z%T@y?$3?AJi-uzJIM$K?4{JyC@D$Q z<~k-q9&DU`TN@LFW(+SZ!q<%o#e`g5JaPJ)9@FIASQc4KH2`B=Fq^DaL?$9;$H+#~?}XMVP`xNjyojBpGS z;*ODbik7qc9pWy(q6lR9hWvt=3m+PoCwxK{ls^oxtHVzLqzjB$olE$=pB558?2h0N zy^UJJV^DsHeF4in$sFE3~GeaUz@OJ_rFmK zQW=V~V1vEcN}E7ljU2rMXlO;QOykkXC$Be(0KNwl3(xWxp8632V3*EvVU`Wzp>2%> zy}XVf2KzYYkAzqqC1X7xxO zHh;kfv&{=aBJa3!+CkWzZDF=3Ve>zh?1)T@6II1F&f-Nxzvi|MUcT zlztP@qep=eC6hw0SHZ3?FVkRY1gDq>P8POq@dAUnn{Q6iVaT@_?IyFGYcS7$gG2O< zOk-Jvl;|-9J2nr;@9dQ$GP=?%9(fqmT}z717edl5F4e}n1(x$QJ+`+h$%v$|$0x#W z3ZcU7#||JzIa&qcPJS*Ild{vqZZVQdst{I)WKxXHg}^dXF-*p#NMxX2~aXx zf0xDR9GJ{Iy;xE67non3r)&&IS!w>}yK{yyOOZ|$1b_JOL=&plc*C^%6ZiH6Z$y($g2VF8dB zUSKu51iK@Y%G`+9ltIou-$9}A1-P)Yu#ZlZJQsZ^}sIyVOHfy+!E#@1*} zUll|$a}+r@gmdYWS?>feBk`6YlmAM~SgUur8cII%v#Wb>;auz}R(3mm&2mNM zTZNT%rNyIk*l5+cU}wBb85aVAo1}`;E%Y&xn4|B7B-2tX+URN!LD0m@&?}Us%4ayk z(wVKn(V}z_lqqLwo0=Zr5=1jd;1tbo^GyqBDkt=)`~pBXz!IH5k$7!$9gVwukdCA% zQiqX=h2Yn0R|?yu<(&`kh?1kacsJhA&64+Sxo!Ey#0pml&|eL2jrT$Y#YATuZW#t9 zm>0#>c;xB5Q=}$ipuOg7u=X26wPxa9;K&%u))+`n(m^Wr<>`}4CVYk}g}V);W^1e- zTLs-G{*ylDAx6WPLqOjFVxAS8#I1*HvbvaAZ z^^O;a^ay?NIsFH}p%>y)Ch5ii*l|U*mzZz5Oe-Mt%dyThobz;A5_hyP>>>;hXA^?* zr@W#fV6oHwn|h-n7%|Stw^@4HZc3s=vwCNIky9#@U(Kj*fZ0geuzPebnQx(bu+b|L z#2+|*F%AGxc)UkU^6k=$v;m|ZJFyL?0pFCnt+-buBMaA57MFg6vVFn(a=#6dUTvAv zsL*NC2dTuUIvM|bt~^{x(4-!mdd4Ajd-IP~nXWo?9&+y^W?grtnXtHeIR} zdnSrikZ|>wRP6BMlih>$3*U{D@kZ#Mx+#C&J1ytX06+=VHzqlMkQ`(iEdtdOf3?*gOCO-nG6V8(@P<%R8c z=lpB{zFNQpNSWqQ7WgRW^SnSf+!3*8nOkK@b|CUBaO-`2&eataIP&Q&K#)W^{Y480 z2$E8l=hGT&uy=SkCDMdMKgj`jv69~IYf&h{Ac4_-fbR;9Z1PpvIIQBF(loJI*V=%c zhMJ1A#XMB@7u3@>zp)oIsG|g(mZBDY`#=o{4Hz}PhU9U4Shsaf=KF#!-{8H?_UL0P zj9Q4eI`~|+2<_qE>$!l7>L%q1OIAg#F;}{k^VyiO+@K92AJan0I5gHX%QE(C=!-Dj zH1;U!X@4Z#8jXJrNBRWEFj%Lk=O2I>P9KSXs(^{0TJw@I^*oj;+eYXC?JqOl%_^$x zV})M9ZV0qe+@X@y?m*nQRCo1u*Qv0_l1{f#B96r=?#K-SkHeU5^;mhg=BLmzGkk5Q zeW$!oidu_|SXcXt2r&>a%tp)6Yk}?zs5Ywx0axWcBk-Lma-Lb+`6i&Zn>b%?iR(YB zVT};wF_+}UcA3UAbW0O7rYs(FC&A+lBy9HJ+(nTJGa)h%$+K5UbvEkm>0x}B$xaB@ zHTMG|AL&kjJ{^q*`n!CbS=U#>V)^zL@eUkbV zjM*to=O=E{rR|s-1X|6OJyADX$F8zB|oTtwu+ud7afb?$XopW zl8=;!e{BF&|C}7!A|WR`Ftgd$(+~@9Q=3N&Ku7b4r0fCA;qK`=q#>>Y%uvwFy>Q>Z z3v0UF@Mp;h^aHcaWSLfms*%YgC2`3SfZV9$R$(5yxP^kW8%yUpV!y}qOZLtOSU`oZSmgUw4HzYd%TO9Ey+k$Vwo%0Rotgz)vf-IrgdL75F@ z)b9Ja%dbX=Tw3IKG6mOg6>!q!KgZ_bG<6crAJKR7-P&bM4qUc1e_%A|11U!<-rl z87mpvloczIZ%N&$1u z5Pxt5Zc^wr0ORMC19Q;<=0fEJ7S&f)Xl}KBf1Y+(;RXSfy7zBc7VvoMt$z;rnxI@f zA}Z{0`hCyjWR0|ei}J+#m2>xa7fa)NXKTw}(9na83sV_F!6Z+O3^As8d%wQw8{lfm z2o=GRL1H5CXFzvEA3WL-9|38yz#hQ;`vaV>dlsOjsh?z^fh@ZF$287^4e--`(uQe$ zLtpZhJ3q&0iE2J4od)fDj}F(q57u}xqEr6>zGwZSlToA}6sCA;#$jR>5I&*6u@+qa z6z(PKW)z5)oTLDDX@|B;@E|Nz1DGC=tJ%HOD*%|`x8*55u){zCloVhKa0Zy~yUhDe zn`Hn+%jA%+TY-yAmPIOxzkFvi6*<(`pFC5tn~`WCfM0Ny?S;0LQ4t7>Be>K$Np0(^rUpCz@(h4 zGyr4gd)z+b6*Re5IoJFRwh`cUUwrL&R}mYHeF!|*O4ZG8X4howFEhFbs&k;L6I^Od zLA(ZZ5v7)B8*zNS`MXgV<(Qr{08n>98Q)Dr^SG>+fFdzaR=p^HX+-mD`#uRKt=l+b z-3;=SHQ(Nd)epYU;(?5}sC}kg+8YYa1t5Vb`0??T+wi?9 zBL}9>?fYYhfYu(!ndD9HcZrZ4aIJxRg?!!Hehv%}eYQN0kOmm>S)noS@PFKyWyi1a zMxk|o7+>s9A}0+j`a$yj5lAkP-T%^ZwK&6?5TtRhih+!)CrAXcN5|nY2Vh;z@u6_* zIG8HLwoO(OQ8F)@1qhG5QPBmo6lO2tYF^x|L!Pu!AR4|h<5EAm3*Z1&KwnhFykQoN zVE{6q4_9)0r;IXyu4J;U(s-_PEkF~5ttGk$c`V9RG|o8%0HIcoK61sy*6pg^=9Pq4 zVv(84bOrujKNfIkiir&F03?ARGo_>oPlGNpCb6pK_m_vkg;4#O*i2UP5n+nbfpUm$r7SmDEo47Vmy8&I_RcbnO)!dpnTeMm%zSYBr zdJurj(1Oo2|NPQ+T8gx+v*~+8Zb`>hZXj{81Jn`+Y6qEPflMl!Qa=B=I-pm}?qvZo zckEVP07kU*xqf^A;A1k_owjJ!Mno5alp>UY?!^H{6OdAs|K?lQPp_XUQLIHWxOw2@ z>+`UYfs$C%=BX~ecF+9$aCNoa1{?1pfJc6T_Q;tg_mZne>S?OJV0>fG0iVMtC~V?j z!{$7cMVU+s;0G%#X|DC7B8LDdb9z^KmsIxTkPWTz3#>{k;B)Pu$DCtH{LSS#=W3=E z_l1yC=`@RAC=s)in9>MZLl`034)jFJR}b=lK=6$>R!<`?^#D!f*~cCQxXK_C{Wh_h ze3I`(Gy%FlDry7%BgUn1S+E3ij0~(5tu+9bw3gG>iatcVIe|>m@H}vQcDWrMzKLiN z1B$8;apy~6Y!VC%Mz=p^164^)ew0>Mz|_#QSYC-aXMU+4;geSrWlzKa9IE{dmlO!U z7p)%9L3j_1Ns?&}wn0KuX^eNtS*}739rpKD4gPgt!00t}CFENNFrnmW?|?5#V2Le6 zm${`XbT{7=D85aUte!}H;1i*CR@NhO$9L4Z1dd#%Q+~*Rma$j=r5t3zyws$52f*WQ|es3=`T~G78Z^E6Q!u@F=j{gy+RixWs?jrt0Y{ zaor5)05~kf0wga8xq4AXYCrsRj9uBEfa?32K`3wY;=@%H>6BM}M5HNelap8KCh{!s zm;3t=0A`jKC$tn!sUV`Wm#Kp}16TGZ-tcHEK|c#ICOkl!0lGBH-^>8Rovlg~chvWn zFBx#6B7${xgx=itsT>242BKg{Ml^hkCi(iYF=S0jCd0U4FUrcE>=HfIfvsm%6iikh@=o&@an|5r24N(^4hF-diO)!CX&M326&q` z=Szd6AuNu1kCHkIsZ1UqA{MHd-6gy1nR_7%+!h+84ii|WHJ@ws2vV@JOX;Y6aGWE} zw~vtPFVw$I#k-4pRY!v(ZZ)gqbKOF>sUfFAt0rNWngQuIfoz6@ezE#bFd0pnid%8t z?o`igsqh|yH=>In3By%Zvo-*`WA_8-mtliw9{8lboDWA5s@ADeq20CW}ohxvs7 zy?kx6Ps#N^?fmTQgp+5c(*Gnl_x(*`i*6rbh7MsyL0KR*Gm9kiU#pg?gp z;(vbSyav>`lqEd1O@5E}KLf{%N(}pNx7FoENA|xB>OT#^oXCJa1^xbY?%$L7Ywl-l zJ-G}jTu|;#=>2OC^tjKI@;zGT-aA+C`Lu6cC{}YYs6O%e{Pk^do2Thf+lx1}!r0bp zV>c`M%9ur|D4p4<-Ve4XH}?^wi?k}X3^LSGlqbk6)^(cB4OOoiAh~}ryHMeM`<%Q` zuVDf{jDw%fc4d*iPJMV#ul1IQRw7>@JmHtwoR4UNN=Uj0eSv~p#k}~VRKv-YNNel( zaO^pP2igH71rwhghW;3Yn0RGvtyUvXNx^!Us6Uh!X-{RP@6b|#cO|8-Q)(B?+v2&&KgOVgmH4KO#q zt-|Ji9hDZk<(L-(ATEuO{QJjOP>=?go20L{>A#MO6SPcq$9UyW8^$aoP@AOSEXt9% z2Ew8LoGSW9O)AiG3~Qz5KMeRUOH@_?oA6dWhYH}0{~Q(U)P0ad84C-G?9qX{Yb=fx zK?*8aj$={odEpw_Ww}tSbJ)1}zrSh1CP{g}Npi`4k@cx^KPyn%eL2z)a5J4e$8GQ2sIV)KHuv zc-rkhI-DmI({wA*$^K*JrMQfosV^A+=x|ZL8|EdMv;4=*|8J@PZ>i7B`~NSi=8x_? WIoSz)auK)|ETuc@@>RDiAN~*ZIzN;E literal 0 HcmV?d00001 diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..bf845cb --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,3 @@ +# Roadmap + +This document outlines the development roadmap for the node-resource-manager project. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..270ae8e --- /dev/null +++ b/go.mod @@ -0,0 +1,23 @@ +module github.com/openyurtio/node-resource-manager + +go 1.15 + +require ( + github.com/aliyun/alibaba-cloud-sdk-go v1.61.895 + github.com/golang/mock v1.3.1 + github.com/imdario/mergo v0.3.11 // indirect + github.com/sirupsen/logrus v1.7.0 + github.com/stretchr/testify v1.6.1 + golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect + golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect + gopkg.in/yaml.v2 v2.4.0 + k8s.io/api v0.20.2 + k8s.io/apimachinery v0.20.2 + k8s.io/client-go v11.0.0+incompatible + k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 +) + +replace ( + k8s.io/api => k8s.io/api v0.19.2 + k8s.io/client-go => k8s.io/client-go v0.19.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..74780d2 --- /dev/null +++ b/go.sum @@ -0,0 +1,388 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.895 h1:8OvEa8rSbkUzjBNTj3vV9mJx+i8Tw6YS1J26FyBURDw= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.895/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms= +k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= +k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= +k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= +k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/main.go b/main.go new file mode 100644 index 0000000..052179a --- /dev/null +++ b/main.go @@ -0,0 +1,123 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "io" + "os" + "strings" + "time" + + "github.com/openyurtio/node-resource-manager/pkg/manager" + "github.com/openyurtio/node-resource-manager/pkg/signals" + log "github.com/sirupsen/logrus" +) + +const ( + // LogfilePrefix prefix of log file + LogfilePrefix = "/var/log/openyurt/" + // MBSIZE MB size + MBSIZE = 1024 * 1024 +) + +// BRANCH is NRM Branch +var _BRANCH_ = "" + +// VERSION is NRM Version +var _VERSION_ = "" + +// BUILDTIME is NRM Buildtime +var _BUILDTIME_ = "" + +var ( + nodeID = flag.String("nodeid", "", "node id") + cmName = flag.String("cm-name", "node-resource-manager", "used configmap name") + cmNameSpace = flag.String("cm-namespace", "kube-system", "used configmap namespace") + logLevel = flag.String("log-level", "Info", "Set Log Level") + updateInterval = flag.Int("update-interval", 30, "Node Storage update internal time(s)") + masterURL = flag.String("master", "", "The address of the Kubernetes API server (https://hostname:port, overrides any value in kubeconfig)") + kubeconfig = flag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information") +) + +// main +func main() { + flag.Parse() + + // set log config + setLogAttribute("unified-resource-manager") + log.Infof("Unified Resource Manager, Version: %s, Build Time: %s", _VERSION_, _BUILDTIME_) + + // new signal handler + stopCh := signals.SetupSignalHandler() + + // New Controller Manager + manager := manager.NewManager(*nodeID, *cmName, *cmNameSpace, *updateInterval, *masterURL, *kubeconfig) + manager.Run(stopCh) + + os.Exit(0) +} + +func init() { + flag.Set("logtostderr", "true") +} + +// rotate log file by 2M bytes +// default print log to stdout and file both. +func setLogAttribute(driver string) { + logType := os.Getenv("LOG_TYPE") + logType = strings.ToLower(logType) + if logType != "stdout" && logType != "host" { + logType = "both" + } + if logType == "stdout" { + return + } + + os.MkdirAll(LogfilePrefix, os.FileMode(0755)) + logFile := LogfilePrefix + driver + ".log" + f, err := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + os.Exit(1) + } + + // rotate the log file if too large + if fi, err := f.Stat(); err == nil && fi.Size() > 2*MBSIZE { + f.Close() + timeStr := time.Now().Format("-2006-01-02-15:04:05") + timedLogfile := LogfilePrefix + driver + timeStr + ".log" + os.Rename(logFile, timedLogfile) + f, err = os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + os.Exit(1) + } + } + if logType == "both" { + mw := io.MultiWriter(os.Stdout, f) + log.SetOutput(mw) + } else { + log.SetOutput(f) + } + + logLevelLow := strings.ToLower(*logLevel) + if logLevelLow == "debug" { + log.SetLevel(log.DebugLevel) + } else if logLevelLow == "warning" { + log.SetLevel(log.WarnLevel) + } + log.Infof("Set Log level to %s...", logLevelLow) +} diff --git a/pkg/config/cloud.go b/pkg/config/cloud.go new file mode 100644 index 0000000..54df3d3 --- /dev/null +++ b/pkg/config/cloud.go @@ -0,0 +1,104 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/openyurtio/node-resource-manager/pkg/utils" + log "github.com/sirupsen/logrus" +) + +func newEcsClient(accessKeyID, accessKeySecret, accessToken string) (ecsClient *ecs.Client) { + var err error + regionID, _ := utils.GetMetaData(RegionIDTag) + if accessToken == "" { + ecsClient, err = ecs.NewClientWithAccessKey(regionID, accessKeyID, accessKeySecret) + if err != nil { + return nil + } + } else { + ecsClient, err = ecs.NewClientWithStsToken(regionID, accessKeyID, accessKeySecret, accessToken) + if err != nil { + return nil + } + } + return +} + +// GetDefaultAK read default ak from local file or from STS +func GetDefaultAK() (string, string, string) { + accessKeyID, accessSecret := GetLocalAK() + + accessToken := "" + if accessKeyID == "" || accessSecret == "" { + accessKeyID, accessSecret, accessToken = GetSTSAK() + } + + return accessKeyID, accessSecret, accessToken +} + +func GetLocalAK() (string, string) { + var accessKeyID, accessSecret string + // first check if the environment setting + accessKeyID = os.Getenv("ACCESS_KEY_ID") + accessSecret = os.Getenv("ACCESS_KEY_SECRET") + if accessKeyID != "" && accessSecret != "" { + return accessKeyID, accessSecret + } + + return accessKeyID, accessSecret +} + +// RoleAuth define STS Token Response +type RoleAuth struct { + AccessKeyID string + AccessKeySecret string + Expiration time.Time + SecurityToken string + LastUpdated time.Time + Code string +} + +// GetSTSAK get STS AK and token from ecs meta server +func GetSTSAK() (string, string, string) { + roleAuth := RoleAuth{} + subpath := "ram/security-credentials/" + roleName, err := utils.GetMetaData(subpath) + if err != nil { + log.Errorf("GetSTSToken: request roleName with error: %s", err.Error()) + return "", "", "" + } + + fullPath := filepath.Join(subpath, roleName) + roleInfo, err := utils.GetMetaData(fullPath) + if err != nil { + log.Errorf("GetSTSToken: request roleInfo with error: %s", err.Error()) + return "", "", "" + } + + err = json.Unmarshal([]byte(roleInfo), &roleAuth) + if err != nil { + log.Errorf("GetSTSToken: unmarshal roleInfo: %s, with error: %s", roleInfo, err.Error()) + return "", "", "" + } + return roleAuth.AccessKeyID, roleAuth.AccessKeySecret, roleAuth.SecurityToken +} diff --git a/pkg/config/global.go b/pkg/config/global.go new file mode 100644 index 0000000..a9a76ca --- /dev/null +++ b/pkg/config/global.go @@ -0,0 +1,71 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "context" + + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +// GlobalConfig save global values for plugin +type GlobalConfig struct { + //EcsClient *ecs.Client + //Region string + NodeInfo *v1.Node + KubeClient *kubernetes.Clientset +} + +var ( + GlobalConfigVar GlobalConfig +) + +const ( + // RegionIDTag is region id + RegionIDTag = "region-id" + // InstanceIDTag is instance id + InstanceIDTag = "instance-id" +) + +// GlobalConfigSet set Global Config +func GlobalConfigSet(nodeID, masterURL, kubeconfig string) { + + // Global Configs Set + cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) + if err != nil { + log.Fatalf("Error building kubeconfig: %s", err.Error()) + } + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + log.Fatalf("Error building kubernetes clientset: %s", err.Error()) + } + + node, err := kubeClient.CoreV1().Nodes().Get(context.Background(), nodeID, metav1.GetOptions{}) + if err != nil { + log.Fatal("Error get current node info: %s", err.Error()) + } + + // Global Config Set + GlobalConfigVar = GlobalConfig{ + KubeClient: kubeClient, + NodeInfo: node, + } +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go new file mode 100644 index 0000000..df583ec --- /dev/null +++ b/pkg/manager/manager.go @@ -0,0 +1,113 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "context" + "time" + + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/manager/memory" + "github.com/openyurtio/node-resource-manager/pkg/manager/quotapath" + "github.com/openyurtio/node-resource-manager/pkg/manager/volumegroup" + log "github.com/sirupsen/logrus" + "k8s.io/client-go/kubernetes" +) + +// Manager interface define local resource manager's action +type Manager interface { + AnalyseConfigMap() error + ApplyResourceDiff() error +} + +// UnifiedResourceManager is global resource manager struct +type UnifiedResourceManager struct { + KubeClientSet *kubernetes.Clientset + UpdateInterval int + NodeID string +} + +// NewDriver create a cpfs driver object +func NewManager(nodeID, cmName, cmNameSpace string, updateInterval int, masterURL, kubeconfig string) *UnifiedResourceManager { + manager := &UnifiedResourceManager{ + NodeID: nodeID, + UpdateInterval: updateInterval, + } + // Config GlobalVar + config.GlobalConfigSet(nodeID, masterURL, kubeconfig) + + return manager +} + +// Run Start the UnifiedResourceManager +// First will check UnifiedResource exist or not; +// Update UnifiedResource every internal seconds, the total/free capacity, storage status; +// Maintain Unified Resource, Like: local disk in alibaba cloud cluster. +func (urm *UnifiedResourceManager) Run(stopCh <-chan struct{}) { + ctx := context.Background() + + // Create UnifiedResource CR if not exist + urm.CreateUnifiedResourceCRD(ctx) + + // Maintain VolumeGroup if set in configMap + go urm.BuildUnifiedResource() + + // UpdateUnifiedStorage + // go wait.Until(urm.RecordUnifiedResources, time.Duration(urm.UpdateInterval)*time.Second, stopCh) + + log.Infof("Starting to update node storage on %s...", urm.NodeID) + <-stopCh + log.Infof("Stop to update node storage...") +} + +// CreateUnifiedResourceCRD ... +func (urm *UnifiedResourceManager) CreateUnifiedResourceCRD(ctx context.Context) error { + return nil +} + +// BuildUnifiedResource ... +func (urm *UnifiedResourceManager) BuildUnifiedResource() { + log.Infof("BuildUnifiedResource:: Starting to maintain unified storage...") + rms := []Manager{volumegroup.NewResourceManager(), quotapath.NewResourceManager(), memory.NewResourceManager()} + + for { + for _, rm := range rms { + err := BuildResource(rm) + if err != nil { + continue + } + } + time.Sleep(time.Duration(20) * time.Second) + } +} + +// BuildResource ... +func BuildResource(m Manager) error { + + // Get Desired VolumeGroup from ConfigMap + err := m.AnalyseConfigMap() + if err != nil { + return err + } + return m.ApplyResourceDiff() + +} + +// Update Unified Storage CRD every internal seconds +func (urm *UnifiedResourceManager) RecordUnifiedResources() { + // Get Unified Storage Object +} diff --git a/pkg/manager/memory/memory.go b/pkg/manager/memory/memory.go new file mode 100644 index 0000000..721c058 --- /dev/null +++ b/pkg/manager/memory/memory.go @@ -0,0 +1,114 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memory + +import ( + "io/ioutil" + "os" + + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/utils" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +// ResourceManager ... +type ResourceManager struct { + Memory []*MConfig + pmem utils.Pmemer + configPath string +} + +// NewResourceManager ... +func NewResourceManager() *ResourceManager { + return &ResourceManager{ + Memory: []*MConfig{}, + pmem: utils.NewNodePmemer(), + configPath: "/etc/unified-config/memory", + } +} + +// AnalyseConfigMap analyse memory resource config +func (mrm *ResourceManager) AnalyseConfigMap() error { + memoryConfig := []*MConfig{} + memoryList := &MList{} + + yamlFile, err := ioutil.ReadFile(mrm.configPath) + if err != nil { + if os.IsNotExist(err) { + log.Debugf("memory config file %s not exist", mrm.configPath) + return nil + } + log.Errorf("AnalyseConfigMap:: yamlFile.Get memory error %v", err) + return err + } + err = yaml.Unmarshal(yamlFile, memoryList) + if err != nil { + log.Errorf("AnalyseConfigMap:: parse yaml file error: %v", err) + return err + } + + nodeInfo := config.GlobalConfigVar.NodeInfo + for _, memConfig := range memoryList.Memories { + isMatched := utils.NodeFilter(memConfig.Operator, memConfig.Key, memConfig.Value, nodeInfo) + if isMatched { + conf := &MConfig{} + if len(memConfig.Topology.Regions) != 1 { + log.Errorf("AnalyseConfigMap:: regions has multi config %s", memConfig.Topology.Regions) + continue + } + conf.Region = memConfig.Topology.Regions[0] + conf.Type = memConfig.Topology.Type + memoryConfig = append(memoryConfig, conf) + } + } + mrm.Memory = memoryConfig + return nil +} + +// ApplyResourceDiff apply memory resource to current node +func (mrm *ResourceManager) ApplyResourceDiff() error { + log.Infof("ApplyResourceDiff: matched node resources mrm.Memory: %v", mrm.Memory) + for _, memConfig := range mrm.Memory { + devicePath, _, err := mrm.pmem.GetPmemNamespaceDeivcePath(memConfig.Region, "devdax") + if err != nil { + err := mrm.pmem.CreateNamespace(memConfig.Region, "dax") + if err != nil { + log.Errorf("applyResourceDiff:: create kmem namespace for region [%s], error: %v", memConfig.Region, err) + continue + } + devicePath, _, err = mrm.pmem.GetPmemNamespaceDeivcePath(memConfig.Region, "devdax") + if err != nil { + log.Errorf("applyResourceDiff:: list kmem namespace for region [%s], error: %v", memConfig.Region, err) + continue + } + } + isCreated, err := mrm.pmem.CheckKMEMCreated(devicePath[5:]) + if err != nil { + log.Errorf("applyResourceDiff:: check kmem create error: %v", err) + continue + } + if !isCreated { + err := mrm.pmem.MakeNamespaceMemory(devicePath[5:]) + if err != nil { + log.Errorf("applyRegionQuotaPath:: make kmem memory failed %v", err) + continue + } + } + } + return nil +} diff --git a/pkg/manager/memory/memory_test.go b/pkg/manager/memory/memory_test.go new file mode 100644 index 0000000..7149e38 --- /dev/null +++ b/pkg/manager/memory/memory_test.go @@ -0,0 +1,127 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memory + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/golang/mock/gomock" + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/model" + "github.com/openyurtio/node-resource-manager/pkg/utils" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func makeValidResourceYaml() *model.ResourceYaml { + return &model.ResourceYaml{ + Name: "foo", + } +} + +func makeResourceYamlCustom(tweaks ...func(*model.ResourceYaml)) *model.ResourceYaml { + resourceYaml := makeValidResourceYaml() + for _, fn := range tweaks { + fn(resourceYaml) + } + return resourceYaml +} + +// EnsureVolumeGroupEnv ... +func EnsureVolumeGroupEnv() (string, error, *ResourceManager) { + config.GlobalConfigVar.NodeInfo = &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{"bar": "foo"}, + }, + } + configPath := "/tmp/memory" + mdkirCmd := "mkdir" + _, err := exec.LookPath(mdkirCmd) + if err != nil { + if err == exec.ErrNotFound { + return "", fmt.Errorf("EnsureFolder:: %q executable not found in $PATH", mdkirCmd), nil + } + return "", err, nil + } + + mkdirArgs := []string{"-p", filepath.Dir(configPath)} + //log.Infof("mkdir for folder, the command is %s %v", mdkirCmd, mkdirArgs) + _, err = exec.Command(mdkirCmd, mkdirArgs...).CombinedOutput() + if err != nil { + return "", fmt.Errorf("EnsureFolder:: mkdir for folder error: %v", err), nil + } + + newMockVolumegroupResourceManager := func() *ResourceManager { + return &ResourceManager{ + configPath: configPath, + } + } + return configPath, nil, newMockVolumegroupResourceManager() +} + +func TestAnalyseConfigMap(t *testing.T) { + mockCtl := gomock.NewController(t) + defer mockCtl.Finish() + configPath, err, resourceManager := EnsureVolumeGroupEnv() + if err != nil { + t.Fatal(err) + } + defer os.Remove(configPath) + mockPmemer := utils.NewMockPmemer(mockCtl) + resourceManager.pmem = mockPmemer + setOpInOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpIn + m.Value = "foo" + } + + setPmemTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "pmem", + Regions: []string{"region0"}, + } + } + testYamls := MList{Memories: []model.ResourceYaml{ + *makeResourceYamlCustom(setOpInOperatorElement, setPmemTopology), + }} + + d, err := yaml.Marshal(&testYamls) + if err != nil { + t.Error() + } + err = ioutil.WriteFile(configPath, d, 0777) + if err != nil { + t.Fatal(err) + } + + assert.Nil(t, resourceManager.AnalyseConfigMap()) + assert.Equal(t, 1, len(resourceManager.Memory)) + gomock.InOrder( + mockPmemer.EXPECT().GetPmemNamespaceDeivcePath(gomock.Eq("region0"), gomock.Eq("devdax")).Return("/dev/dax0.0", "", nil), + mockPmemer.EXPECT().CheckKMEMCreated(gomock.Eq("dax0.0")).Return(true, nil), + ) + assert.Nil(t, resourceManager.ApplyResourceDiff()) + +} diff --git a/pkg/manager/memory/type.go b/pkg/manager/memory/type.go new file mode 100644 index 0000000..a1908af --- /dev/null +++ b/pkg/manager/memory/type.go @@ -0,0 +1,32 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memory + +import ( + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +// MConfig ... +type MConfig struct { + Type string + Region string +} + +// MList ... +type MList struct { + Memories []model.ResourceYaml `yaml:"memory,omitempty"` +} diff --git a/pkg/manager/quotapath/quotapath.go b/pkg/manager/quotapath/quotapath.go new file mode 100644 index 0000000..4f21c86 --- /dev/null +++ b/pkg/manager/quotapath/quotapath.go @@ -0,0 +1,175 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quotapath + +import ( + "io/ioutil" + "os" + "strings" + + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/utils" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +// ResourceManager ... +type ResourceManager struct { + DeviceQuotaPath map[string]*QpConfig + RegionQuotaPath map[string]*QpConfig + mounter utils.Mounter + mkfsOption []string + pmemer utils.Pmemer + configPath string +} + +// NewResourceManager ... +func NewResourceManager() *ResourceManager { + return &ResourceManager{ + DeviceQuotaPath: make(map[string]*QpConfig), + RegionQuotaPath: make(map[string]*QpConfig), + mounter: utils.NewMounter(), + pmemer: utils.NewNodePmemer(), + configPath: "/etc/unified-config/quotapath", + } +} + +// AnalyseConfigMap analyse quotapath resource config +func (qrm *ResourceManager) AnalyseConfigMap() error { + deviceQuotaConfig := map[string]*QpConfig{} + regionQuotaConfig := map[string]*QpConfig{} + quotaPathList := &QPList{} + yamlFile, err := ioutil.ReadFile(qrm.configPath) + if err != nil { + if os.IsNotExist(err) { + log.Debugf("quota config file %s not exist", qrm.configPath) + return nil + } + log.Errorf("AnalyseConfigMap:: yamlFile.Get error %v", err) + return err + } + err = yaml.Unmarshal(yamlFile, quotaPathList) + if err != nil { + log.Errorf("AnalyseConfigMap:: parse yaml file error: %v", err) + return err + } + mountPathMap := map[string]string{} + nodeInfo := config.GlobalConfigVar.NodeInfo + for _, quotaConfig := range quotaPathList.QuotaPaths { + isMatched := utils.NodeFilter(quotaConfig.Operator, quotaConfig.Key, quotaConfig.Value, nodeInfo) + if isMatched { + if _, ok := mountPathMap[quotaConfig.Name]; ok { + log.Errorf("AnalyseConfigMap:: quotapath config has multi mount path config [%s] on same node", quotaConfig.Name) + continue + } + switch quotaConfig.Topology.Type { + case "device": + conf := &QpConfig{} + if len(quotaConfig.Topology.Devices) != 1 { + log.Errorf("AnalyseConfigMap:: quotapath regions [%v] config only support one device", quotaConfig.Topology.Regions) + continue + } + conf.Device = quotaConfig.Topology.Devices[0] + conf.Fstype = quotaConfig.Topology.Fstype + conf.Options = quotaConfig.Topology.Options + conf.Type = quotaConfig.Topology.Type + deviceQuotaConfig[quotaConfig.Name] = conf + case "pmem": + conf := &QpConfig{} + if len(quotaConfig.Topology.Regions) != 1 { + log.Errorf("AnalyseConfigMap:: quotapath regions [%s] config only support one device", quotaConfig.Topology.Regions) + continue + } + conf.Region = quotaConfig.Topology.Regions[0] + conf.Fstype = quotaConfig.Topology.Fstype + conf.Options = quotaConfig.Topology.Options + conf.Type = quotaConfig.Topology.Type + regionQuotaConfig[quotaConfig.Name] = conf + default: + log.Errorf("AnalyseConfigMap:: not support quotapath config type: [%v]", quotaConfig.Topology.Type) + continue + } + mountPathMap[quotaConfig.Name] = "" + } + } + + qrm.DeviceQuotaPath = deviceQuotaConfig + qrm.RegionQuotaPath = regionQuotaConfig + return nil +} + +// ApplyResourceDiff apply quotapath resource to current node +func (qrm *ResourceManager) ApplyResourceDiff() error { + log.Infof("ApplyResourceDiff: matched node resources qrm.DeviceQuotaPath: %v, qrm.RegionQuotaPath: %v", qrm.DeviceQuotaPath, qrm.RegionQuotaPath) + qrm.mkfsOption = strings.Split("-O project,quota", " ") + err := qrm.applyDeivceQuotaPath() + if err != nil { + log.Errorf("ApplyResourceDiff:: apply deivce quotapath error: %v", err) + } + err = qrm.applyRegionQuotaPath() + if err != nil { + log.Errorf("ApplyResourceDiff:: apply region quotapath error: %v", err) + } + return err +} + +func (qrm *ResourceManager) applyDeivceQuotaPath() error { + + for mountPath, deivceQuotaPathConfig := range qrm.DeviceQuotaPath { + err := qrm.mounter.EnsureFolder(mountPath) + if err != nil { + log.Errorf("applyDeivceQuotaPath:: ensure quotapath error: %v", err) + continue + } + err = qrm.mounter.FormatAndMount(deivceQuotaPathConfig.Device, mountPath, deivceQuotaPathConfig.Fstype, qrm.mkfsOption, deivceQuotaPathConfig.Options) + if err != nil { + log.Errorf("applyDeivceQuotaPath:: mounter FormatAndMount error: %v", err) + continue + } + } + return nil +} + +func (qrm *ResourceManager) applyRegionQuotaPath() error { + for mountPath, regionQuotaPathConfig := range qrm.RegionQuotaPath { + devicePath, _, err := qrm.pmemer.GetPmemNamespaceDeivcePath(regionQuotaPathConfig.Region, "fsdax") + if err != nil { + if strings.Contains(err.Error(), "list Namespace for region get 0 or multi namespaces") { + err := qrm.pmemer.CreateNamespace(regionQuotaPathConfig.Region, "lvm") + if err != nil { + log.Errorf("applyRegionQuotaPath:: create namespace for region [%s], error: %v", regionQuotaPathConfig.Region, err) + continue + } + devicePath, _, err = qrm.pmemer.GetPmemNamespaceDeivcePath(regionQuotaPathConfig.Region, "fsdax") + } else { + log.Errorf("applyRegionQuotaPath:: get region [%s] namespace device path error: %v", regionQuotaPathConfig.Region, err) + continue + } + } + err = qrm.mounter.EnsureFolder(mountPath) + if err != nil { + log.Errorf("applyRegionQuotaPath:: ensure quotapath error: %v", err) + continue + } + err = qrm.mounter.FormatAndMount(devicePath, mountPath, regionQuotaPathConfig.Fstype, qrm.mkfsOption, regionQuotaPathConfig.Options) + if err != nil { + log.Errorf("applyRegionQuotaPath:: mounter FormatAndMount error: %v", err) + continue + } + } + return nil +} diff --git a/pkg/manager/quotapath/quotapath_test.go b/pkg/manager/quotapath/quotapath_test.go new file mode 100644 index 0000000..03bac62 --- /dev/null +++ b/pkg/manager/quotapath/quotapath_test.go @@ -0,0 +1,151 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quotapath + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/golang/mock/gomock" + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/model" + "github.com/openyurtio/node-resource-manager/pkg/utils" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func makeValidResourceYaml() *model.ResourceYaml { + return &model.ResourceYaml{ + Name: "/tmp/foo", + } +} + +func makeResourceYamlCustom(tweaks ...func(*model.ResourceYaml)) *model.ResourceYaml { + resourceYaml := makeValidResourceYaml() + for _, fn := range tweaks { + fn(resourceYaml) + } + return resourceYaml +} + +// EnsureVolumeGroupEnv ... +func EnsureVolumeGroupEnv() (string, error, *ResourceManager) { + config.GlobalConfigVar.NodeInfo = &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{"bar": "foo"}, + }, + } + configPath := "/tmp/quotapath" + mdkirCmd := "mkdir" + _, err := exec.LookPath(mdkirCmd) + if err != nil { + if err == exec.ErrNotFound { + return "", fmt.Errorf("EnsureFolder:: %q executable not found in $PATH", mdkirCmd), nil + } + return "", err, nil + } + + mkdirArgs := []string{"-p", filepath.Dir(configPath)} + //log.Infof("mkdir for folder, the command is %s %v", mdkirCmd, mkdirArgs) + _, err = exec.Command(mdkirCmd, mkdirArgs...).CombinedOutput() + if err != nil { + return "", fmt.Errorf("EnsureFolder:: mkdir for folder error: %v", err), nil + } + + newMockVolumegroupResourceManager := func() *ResourceManager { + return &ResourceManager{ + configPath: configPath, + } + } + return configPath, nil, newMockVolumegroupResourceManager() +} + +func TestAnalyseDiff(t *testing.T) { + mockCtl := gomock.NewController(t) + defer mockCtl.Finish() + configPath, err, resourceManager := EnsureVolumeGroupEnv() + if err != nil { + t.Fatal(err) + } + defer os.Remove(configPath) + mockPmemer := utils.NewMockPmemer(mockCtl) + mockMounter := utils.NewMockMounter(mockCtl) + resourceManager.mounter = mockMounter + resourceManager.pmemer = mockPmemer + setOpInOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpIn + m.Value = "foo" + } + + setPmemTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "pmem", + Options: "prjquota,shared", + Fstype: "ext4", + Regions: []string{"region0"}, + } + } + setDeviceTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "device", + Options: "prjquota", + Fstype: "ext4", + Devices: []string{"/dev/vdc"}, + } + } + setDeviceNameElement := func(m *model.ResourceYaml) { + m.Name = "/tmp/foo1" + } + testYamls := QPList{QuotaPaths: []model.ResourceYaml{ + *makeResourceYamlCustom(setOpInOperatorElement, setPmemTopology), + *makeResourceYamlCustom(setDeviceNameElement, setOpInOperatorElement, setDeviceTopology), + }} + + d, err := yaml.Marshal(&testYamls) + if err != nil { + t.Error() + } + err = ioutil.WriteFile(configPath, d, 0777) + if err != nil { + t.Fatal(err) + } + + assert.Nil(t, resourceManager.AnalyseConfigMap()) + assert.Equal(t, 1, len(resourceManager.RegionQuotaPath)) + assert.Equal(t, 1, len(resourceManager.DeviceQuotaPath)) + gomock.InOrder( + mockMounter.EXPECT().EnsureFolder( + gomock.Eq("/tmp/foo1")).Return(nil), + mockMounter.EXPECT().FormatAndMount( + gomock.Eq("/dev/vdc"), gomock.Eq("/tmp/foo1"), gomock.Eq("ext4"), gomock.Eq([]string{"-O", "project,quota"}), gomock.Eq("prjquota")).Return(nil), + mockPmemer.EXPECT().GetPmemNamespaceDeivcePath( + gomock.Eq("region0"), gomock.Eq("fsdax")).Return("/dev/pmem0", "", nil), + mockMounter.EXPECT().EnsureFolder( + gomock.Eq("/tmp/foo")).Return(nil), + mockMounter.EXPECT().FormatAndMount( + gomock.Eq("/dev/pmem0"), gomock.Eq("/tmp/foo"), gomock.Eq("ext4"), gomock.Eq([]string{"-O", "project,quota"}), gomock.Eq("prjquota,shared")).Return(nil), + ) + assert.Nil(t, resourceManager.ApplyResourceDiff()) +} diff --git a/pkg/manager/quotapath/type.go b/pkg/manager/quotapath/type.go new file mode 100644 index 0000000..72c791a --- /dev/null +++ b/pkg/manager/quotapath/type.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quotapath + +import ( + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +// QpConfig ... +type QpConfig struct { + Type string + Options string + Fstype string + Region string + Device string +} + +// QPList ... +type QPList struct { + QuotaPaths []model.ResourceYaml `yaml:"quotapath,omitempty"` +} diff --git a/pkg/manager/volumegroup/recorder.go b/pkg/manager/volumegroup/recorder.go new file mode 100644 index 0000000..b2bd271 --- /dev/null +++ b/pkg/manager/volumegroup/recorder.go @@ -0,0 +1,17 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumegroup diff --git a/pkg/manager/volumegroup/type.go b/pkg/manager/volumegroup/type.go new file mode 100644 index 0000000..a3d5dce --- /dev/null +++ b/pkg/manager/volumegroup/type.go @@ -0,0 +1,32 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumegroup + +import ( + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +// VgDeviceConfig ... +type VgDeviceConfig struct { + Name string `json:"name,omitempty"` + PhysicalVolumes []string `json:"physicalVolumes,omitempty"` +} + +// VgList ... +type VgList struct { + VolumeGroups []model.ResourceYaml `yaml:"volumegroup,omitempty"` +} diff --git a/pkg/manager/volumegroup/utils.go b/pkg/manager/volumegroup/utils.go new file mode 100644 index 0000000..0b212f2 --- /dev/null +++ b/pkg/manager/volumegroup/utils.go @@ -0,0 +1,270 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumegroup + +import ( + "crypto/aes" + "crypto/cipher" + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/openyurtio/node-resource-manager/pkg/utils" + log "github.com/sirupsen/logrus" +) + +const ( + // MetadataURL is metadata server url + MetadataURL = "http://100.100.100.200/latest/meta-data/" + // InstanceID is the instance id tag + InstanceID = "instance-id" + // RegionIDTag is the region id tag + RegionIDTag = "region-id" + // NsenterCmd is the nsenter command + NsenterCmd = "/usr/bin/nsenter --mount=/proc/1/ns/mnt " +) + +const ( + // ConfigPath the secret mount file + ConfigPath = "/var/addon/token-config" +) + +// AKInfo access key info +type AKInfo struct { + // AccessKeyId access key id + AccessKeyID string `json:"access.key.id"` + // AccessKeySecret access key secret + AccessKeySecret string `json:"access.key.secret"` + // SecurityToken security token + SecurityToken string `json:"security.token"` + // Expiration expiration duration + Expiration string `json:"expiration"` + // Keyring key ring + Keyring string `json:"keyring"` +} + +// RoleAuth define STS Token Response +type RoleAuth struct { + AccessKeyID string + AccessKeySecret string + Expiration time.Time + SecurityToken string + LastUpdated time.Time + Code string +} + +func ListDevice(vgName string) []string { + cmd := fmt.Sprintf("%s pvscan | grep \"VG %s\" | awk '{print $2}'", NsenterCmd, vgName) + out, err := utils.Run(cmd) + if err != nil { + devs := make([]string, 0) + return devs + } + + outLines := strings.Split(out, "\n") + deviceList := []string{} + for _, line := range outLines { + line = strings.TrimSpace(line) + if line != "" && !strings.HasPrefix(line, "WARNING") { + deviceList = append(deviceList, line) + } + } + return deviceList +} + +func diffDevice(devList1, devList2 []string) bool { + if len(devList1) != len(devList2) { + return true + } + for _, dev1 := range devList1 { + isSearched := false + for _, dev2 := range devList2 { + if dev1 == dev2 { + isSearched = true + break + } + } + if !isSearched { + return true + } + } + + return false +} + +// NewEcsClient create a ecsClient object +func NewEcsClient(accessKeyID, accessKeySecret, accessToken string) (ecsClient *ecs.Client) { + var err error + if accessToken == "" { + ecsClient, err = ecs.NewClientWithAccessKey("cn-hangzhou", accessKeyID, accessKeySecret) + if err != nil { + return nil + } + } else { + ecsClient, err = ecs.NewClientWithStsToken("cn-hangzhou", accessKeyID, accessKeySecret, accessToken) + if err != nil { + return nil + } + } + return +} + +// GetMetaData get host regionid, zoneid +func GetMetaData(resource string) string { + resp, err := http.Get(MetadataURL + resource) + if err != nil { + return "" + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "" + } + return string(body) +} + +// GetDefaultAK read default ak from local file or from STS +func GetDefaultAK() (string, string, string) { + accessKeyID, accessSecret := GetLocalAK() + + accessToken := "" + if accessKeyID == "" || accessSecret == "" { + accessKeyID, accessSecret, accessToken = GetManagedToken() + if accessKeyID != "" { + log.Infof("Get AK: use Managed AK") + } + } else { + log.Infof("Get AK: use Local AK") + } + + if accessKeyID == "" || accessSecret == "" { + accessKeyID, accessSecret, accessToken = GetSTSAK() + } + + return accessKeyID, accessSecret, accessToken +} + +// GetLocalAK read ossfs ak from local or from secret file +func GetLocalAK() (string, string) { + accessKeyID, accessSecret := "", "" + accessKeyID = os.Getenv("ACCESS_KEY_ID") + accessSecret = os.Getenv("ACCESS_KEY_SECRET") + + return strings.TrimSpace(accessKeyID), strings.TrimSpace(accessSecret) +} + +// GetSTSAK get STS AK and token from ecs meta server +func GetSTSAK() (string, string, string) { + roleAuth := RoleAuth{} + subpath := "ram/security-credentials/" + roleName := GetMetaData(subpath) + fullPath := filepath.Join(subpath, roleName) + roleInfo := GetMetaData(fullPath) + + err := json.Unmarshal([]byte(roleInfo), &roleAuth) + if err != nil { + log.Errorf("GetSTSToken: unmarshal roleInfo: %s, with error: %s", roleInfo, err.Error()) + return "", "", "" + } + return roleAuth.AccessKeyID, roleAuth.AccessKeySecret, roleAuth.SecurityToken +} + +// GetManagedToken get ak from csi secret +func GetManagedToken() (string, string, string) { + var akInfo AKInfo + AccessKeyID, AccessKeySecret, SecurityToken := "", "", "" + if _, err := os.Stat(ConfigPath); err == nil { + encodeTokenCfg, err := ioutil.ReadFile(ConfigPath) + if err != nil { + log.Errorf("failed to read token config, err: %v", err) + return "", "", "" + } + err = json.Unmarshal(encodeTokenCfg, &akInfo) + if err != nil { + log.Errorf("error unmarshal token config: %v", err) + return "", "", "" + } + keyring := akInfo.Keyring + ak, err := Decrypt(akInfo.AccessKeyID, []byte(keyring)) + if err != nil { + log.Errorf("failed to decode ak, err: %v", err) + return "", "", "" + } + + sk, err := Decrypt(akInfo.AccessKeySecret, []byte(keyring)) + if err != nil { + log.Errorf("failed to decode sk, err: %v", err) + return "", "", "" + } + + token, err := Decrypt(akInfo.SecurityToken, []byte(keyring)) + if err != nil { + log.Errorf("failed to decode token, err: %v", err) + return "", "", "" + } + layout := "2006-01-02T15:04:05Z" + t, err := time.Parse(layout, akInfo.Expiration) + if err != nil { + log.Errorf("Parse expiration error: %s", err.Error()) + } + if t.Before(time.Now()) { + log.Errorf("invalid token which is expired, expiration as: %s", akInfo.Expiration) + } + AccessKeyID = string(ak) + AccessKeySecret = string(sk) + SecurityToken = string(token) + } + return AccessKeyID, AccessKeySecret, SecurityToken +} + +// PKCS5UnPadding get pkc +func PKCS5UnPadding(origData []byte) []byte { + length := len(origData) + unpadding := int(origData[length-1]) + return origData[:(length - unpadding)] +} + +// Decrypt secret Decrypt +func Decrypt(s string, keyring []byte) ([]byte, error) { + cdata, err := base64.StdEncoding.DecodeString(s) + if err != nil { + log.Errorf("failed to decode base64 string, err: %v", err) + return nil, err + } + block, err := aes.NewCipher(keyring) + if err != nil { + log.Errorf("failed to new cipher, err: %v", err) + return nil, err + } + blockSize := block.BlockSize() + + iv := cdata[:blockSize] + blockMode := cipher.NewCBCDecrypter(block, iv) + origData := make([]byte, len(cdata)-blockSize) + + blockMode.CryptBlocks(origData, cdata[blockSize:]) + + origData = PKCS5UnPadding(origData) + return origData, nil +} diff --git a/pkg/manager/volumegroup/volumegroup.go b/pkg/manager/volumegroup/volumegroup.go new file mode 100644 index 0000000..f1d1c6b --- /dev/null +++ b/pkg/manager/volumegroup/volumegroup.go @@ -0,0 +1,432 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumegroup + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/utils" + log "github.com/sirupsen/logrus" + yaml "gopkg.in/yaml.v2" +) + +const ( + StorageCmNameSpace = "kube-system" + StorageCmName = "cm-node-resource" + VGConfigKey = "volumegroup.json" + AliyunLocalDisk = "aliyun-local-disk" + + VgConfigFile = "/etc/unified-config/volumegroup" + VgTypeDevice = "device" + VgTypePvc = "pvc" + VgTypeLocal = "alibabacloud-local-disk" + VgTypePmem = "pmem" +) + +// ResourceManager ... +type ResourceManager struct { + volumeGroupDeviceMap map[string]*VgDeviceConfig + volumeGroupRegionMap map[string][]string + pmemer utils.Pmemer + lvmer utils.LVM + configPath string +} + +// NewResourceManager ... +func NewResourceManager() *ResourceManager { + return &ResourceManager{ + volumeGroupDeviceMap: make(map[string]*VgDeviceConfig), + volumeGroupRegionMap: make(map[string][]string), + pmemer: utils.NewNodePmemer(), + lvmer: utils.NewNodeLVM(), + configPath: "/etc/unified-config/volumegroup", + } +} + +// DeviceChars ... +var DeviceChars = []string{"b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"} + +// Create VolumeGroup +func (vrm *ResourceManager) createVg(vgName string, desirePvList []string) error { + pvListStr := strings.Join(desirePvList, " ") + tagList := []string{} + out, err := vrm.lvmer.CreateVG(vgName, pvListStr, tagList) + if err != nil { + log.Errorf("createVg:: Create Vg(%s) error with: %s", vgName, err.Error()) + return err + } + log.Infof("createVg:: Successful Create Vg(%s) with out: %s", vgName, out) + return nil +} + +// Upgrade VolumeGroup, only support extend pv; +func (vrm *ResourceManager) updateVg(vgName string, expectPvList, realPvList []string) error { + if len(expectPvList) < len(realPvList) { + msg := fmt.Sprintf("updateVg:: VolumeGroup: %s, expected pv list should be more than current pv list when update volume group: %v, %v", vgName, expectPvList, realPvList) + log.Errorf(msg) + return errors.New(msg) + } + + // removed pv: pv exist in current node, but not in expect + removePv := difference(realPvList, expectPvList) + if len(removePv) > 0 { + msg := fmt.Sprintf("updateVg:: VolumeGroup: %s, expected pv list should be more than current pv list when update volume group: expect %v, current %v, and not support remove pv now", vgName, expectPvList, realPvList) + log.Errorf(msg) + return errors.New(msg) + } + + // added pv: pv exist in expect, but not in current node. + addedPv := difference(expectPvList, realPvList) + if len(addedPv) == 0 { + msg := fmt.Sprintf("updateVg:: VolumeGroup: %s, expected pv list same with current pv list, expect %v, current %v", vgName, expectPvList, realPvList) + log.Errorf(msg) + return errors.New(msg) + } + + pvListStr := strings.Join(addedPv, " ") + _, err := vrm.lvmer.ExtendVG(vgName, pvListStr) + if err != nil { + msg := fmt.Sprintf("updateVg:: Extend vg(%s) error: %v", vgName, err) + log.Errorf(msg) + return errors.New(msg) + } + log.Infof("updateVg:: Successful Add pvs %s to VolumeGroup %s", addedPv, vgName) + return nil +} + +// difference returns the elements in `a` that aren't in `b`. +func difference(a, b []string) []string { + mb := make(map[string]struct{}, len(b)) + for _, x := range b { + mb[x] = struct{}{} + } + var diff []string + for _, x := range a { + if _, found := mb[x]; !found { + diff = append(diff, x) + } + } + return diff +} + +func getPvListForLocalDisk() []string { + // Step 2: Get LocalDisk Number + localDeviceList := []string{} + localDeviceNum, err := getLocalDeviceNum() + if err != nil { + log.Errorf("getPvListForLocalDisk:: LocalDiskMount:: Get Local Disk Number Error, Error: %s", err.Error()) + return localDeviceList + } + if localDeviceNum < 1 { + log.Errorf("getPvListForLocalDisk:: VG not exist and also not local disk exist, localDeivceNum: %v", localDeviceNum) + return localDeviceList + } + + // Step 3: Get LocalDisk device + deviceStartWith := "vdb" + deviceNamePrefix := "vd" + deviceStartIndex := 0 + deviceNameLen := len(deviceStartWith) + if deviceNameLen > 1 { + deviceStartChar := deviceStartWith[deviceNameLen-1 : deviceNameLen] + for index := 0; index < 15; index++ { + if deviceStartChar == DeviceChars[index] { + deviceStartIndex = index + } + } + deviceNamePrefix = deviceStartWith[0 : deviceNameLen-1] + } + for i := deviceStartIndex; i < localDeviceNum; i++ { + deviceName := deviceNamePrefix + DeviceChars[i] + devicePath := filepath.Join("/dev", deviceName) + localDeviceList = append(localDeviceList, devicePath) + } + //log.Infof("doLocalVolumeMounts, Starting LocalDisk Mount: LocalDisk Number: %d, LocalDisk: %v", localDeviceNum, localDeviceList) + return localDeviceList +} + +// Get Local Disk Number from ecs API +// Requirements: The instance must have role which contains ecs::DescribeInstances, ecs::DescribeInstancesType. +func getLocalDeviceNum() (int, error) { + instanceID := GetMetaData(InstanceID) + regionID := GetMetaData(RegionIDTag) + localDeviceNum := 0 + akID, akSecret, token := GetDefaultAK() + client := NewEcsClient(akID, akSecret, token) + + // Get Instance Type + request := ecs.CreateDescribeInstancesRequest() + request.RegionId = regionID + request.InstanceIds = "[\"" + instanceID + "\"]" + instanceResponse, err := client.DescribeInstances(request) + if err != nil { + log.Errorf("getLocalDeviceNum:: Describe Instance: %s Error: %s", instanceID, err.Error()) + return -1, err + } + if instanceResponse == nil || len(instanceResponse.Instances.Instance) == 0 { + log.Infof("getLocalDeviceNum:: Describe Instance Error, with empty response: %s", instanceID) + return -1, err + } + + // Get Instance LocalDisk Number + instanceTypeID := instanceResponse.Instances.Instance[0].InstanceType + instanceTypeFamily := instanceResponse.Instances.Instance[0].InstanceTypeFamily + instanceTypeRequest := ecs.CreateDescribeInstanceTypesRequest() + instanceTypeRequest.InstanceTypeFamily = instanceTypeFamily + response, err := client.DescribeInstanceTypes(instanceTypeRequest) + if err != nil { + log.Errorf("getLocalDeviceNum:: Describe Instance: %s, Type: %s, Family: %s Error: %s", instanceID, instanceTypeID, instanceTypeFamily, err.Error()) + return -1, err + } + for _, instance := range response.InstanceTypes.InstanceType { + if instance.InstanceTypeId == instanceTypeID { + localDeviceNum = instance.LocalStorageAmount + log.Infof("getLocalDeviceNum:: Instance: %s, InstanceType: %s, InstanceLocalDiskNum: %d", instanceID, instanceTypeID, localDeviceNum) + break + } + } + return localDeviceNum, nil +} + +// Get current VolumeGroup in node +// echo vgOjbect contains: vgName, pvList; +func (vrm *ResourceManager) getRealVgList() ([]*VgDeviceConfig, error) { + physicalVolumeList, err := vrm.lvmer.ListPhysicalVolume() + if err != nil { + log.Errorf("List PhysicalVolume get error %v", err) + return nil, err + } + log.Debugf("Real VolumeGroup List: %+v", physicalVolumeList) + nodeVgList := []*VgDeviceConfig{} + for _, physicalVolume := range physicalVolumeList { + isAlreadyAdded := false + for _, nodeVg := range nodeVgList { + if nodeVg.Name == physicalVolume.VgName { + nodeVg.PhysicalVolumes = append(nodeVg.PhysicalVolumes, physicalVolume.Name) + isAlreadyAdded = true + log.Debugf("getRealVgList:: physicalVolumes: %v, physicalVolumeName: %s, volumeGroupName: %v", nodeVg.PhysicalVolumes, physicalVolume.Name, physicalVolume.VgName) + } + } + if isAlreadyAdded == false { + vgPvConfig := &VgDeviceConfig{} + vgPvConfig.Name = physicalVolume.VgName + vgPvConfig.PhysicalVolumes = append(vgPvConfig.PhysicalVolumes, physicalVolume.Name) + nodeVgList = append(nodeVgList, vgPvConfig) + log.Debugf("Add New VolumeGroupPvConfig: %s, %s, %v", physicalVolume.Name, physicalVolume.VgName, vgPvConfig) + } + } + return nodeVgList, nil +} + +// AnalyseConfigMap analyse pmem resource config +func (vrm *ResourceManager) AnalyseConfigMap() error { + + vgDeviceMap := map[string]*VgDeviceConfig{} + vgRegionMap := map[string][]string{} + + volumeGroupList := &VgList{} + yamlFile, err := ioutil.ReadFile(vrm.configPath) + if err != nil { + if os.IsNotExist(err) { + log.Debugf("volume config file %s not exist", vrm.configPath) + return nil + } + log.Errorf("AnalyseConfigMap:: ReadFile: yamlFile.Get error #%v ", err) + return err + } + + err = yaml.Unmarshal(yamlFile, volumeGroupList) + if err != nil { + log.Errorf("AnalyseConfigMap:: Unmarshal: parse yaml file error: %v", err) + return err + } + + nodeInfo := config.GlobalConfigVar.NodeInfo + for _, devConfig := range volumeGroupList.VolumeGroups { + vgDeviceConfig := &VgDeviceConfig{} + + isMatched := utils.NodeFilter(devConfig.Operator, devConfig.Key, devConfig.Value, nodeInfo) + log.Infof("AnalyseConfigMap:: isMatched: %v, devConfig: %+v", isMatched, devConfig) + + if isMatched { + switch devConfig.Topology.Type { + case VgTypeDevice: + vgDeviceConfig.PhysicalVolumes = devConfig.Topology.Devices + vgDeviceMap[devConfig.Name] = vgDeviceConfig + case VgTypeLocal: + tmpConfig := &VgDeviceConfig{} + tmpConfig.PhysicalVolumes = getPvListForLocalDisk() + vgDeviceMap[devConfig.Name] = tmpConfig + case VgTypePvc: + // not support yet + continue + case VgTypePmem: + vgRegionMap[devConfig.Name] = devConfig.Topology.Regions + default: + log.Errorf("AnalyseConfigMap:: Get unsupported volumegroup type: %s", devConfig.Topology.Type) + continue + } + } + } + vrm.volumeGroupDeviceMap = vgDeviceMap + vrm.volumeGroupRegionMap = vgRegionMap + return nil +} + +// ApplyResourceDiff apply volume group resource to current node +func (vrm *ResourceManager) ApplyResourceDiff() error { + + // Get Actual VolumeGroup on node. + actualVgConfig, err := vrm.getRealVgList() + if err != nil { + log.Errorf("ApplyResourceDiff:: Get Node Actual VolumeGroup Error: %s", err.Error()) + return err + } + if len(vrm.volumeGroupDeviceMap) > 0 { + vrm.applyDeivce(actualVgConfig) + } + if len(vrm.volumeGroupRegionMap) > 0 { + vrm.applyRegion(actualVgConfig) + } + + log.Infof("ApplyResourceDiff:: Finish volumegroup loop...") + return nil +} + +func (vrm *ResourceManager) applyDeivce(actualVgConfig []*VgDeviceConfig) error { + // process each expect volume group + for expectVgName, expectVg := range vrm.volumeGroupDeviceMap { + log.Infof("applyDevice:: expectName: %s, expectVgDevices: %v", expectVgName, expectVg.PhysicalVolumes) + isVgExist := false + isVgNeedUpdate := false + realPhysicalVolumeList := []string{} + + for _, realVg := range actualVgConfig { + if expectVgName == realVg.Name { + isVgExist = true + diffs := difference(expectVg.PhysicalVolumes, realVg.PhysicalVolumes) + if len(diffs) != 0 { + realPhysicalVolumeList = realVg.PhysicalVolumes + isVgNeedUpdate = true + } + break + } + } + if !isVgExist { + log.Infof("Create VolumeGroup:: %+v, %+v", expectVgName, expectVg.PhysicalVolumes) + return vrm.createVg(expectVgName, expectVg.PhysicalVolumes) + } else if isVgNeedUpdate { + log.Infof("Update VolumeGroup:: %+v, %+v", expectVgName, expectVg.PhysicalVolumes) + return vrm.updateVg(expectVgName, expectVg.PhysicalVolumes, realPhysicalVolumeList) + } + } + return nil +} + +func (vrm *ResourceManager) applyRegion(actualVgConfig []*VgDeviceConfig) error { + regions, err := vrm.pmemer.GetRegions() + if err != nil { + log.Errorf("applyRegion: get pmem regions error: %v", err) + return err + } + + for _, expectRegions := range vrm.volumeGroupRegionMap { + expectNamespaces := []string{} + for _, expectRegion := range expectRegions { + expectRegionExists := false + for _, region := range regions.Regions { + if expectRegion == region.Dev { + expectRegionExists = true + if len(region.Namespaces) == 0 { + vrm.pmemer.CreateNamespace(region.Dev, "lvm") + } + } + } + if !expectRegionExists { + err := fmt.Errorf("applyRegion:: expect region %s not exists", expectRegion) + return err + } + expectNamespaces = append(expectNamespaces, utils.ConvertRegion2Namespace(expectRegion)) + } + } + updatedRegions, err := vrm.pmemer.GetRegions() + if err != nil { + log.Errorf("applyRegion: get pmem regions error: %v", err) + return err + } + for expectVgName, expectRegions := range vrm.volumeGroupRegionMap { + log.Infof("applyDevice:: expectVgName: %v, expectRegions: %v", expectVgName, expectRegions) + expectLvmInUseDevices := []string{} + expectLvmNotInUseDevices := []string{} + for _, expectRegion := range expectRegions { + devicePath := utils.ConvertNamespace2LVMDevicePath(utils.ConvertRegion2Namespace(expectRegion), updatedRegions) + if devicePath == "" { + log.Errorf("applyRegion:: did not get namespace.Blockdev from expectRegion: %s, regions: %v", expectRegion, updatedRegions) + } + if vrm.pmemer.CheckNamespaceUsed(devicePath) { + log.Warnf("NameSpace heen used region: %v, devicePath: %s", expectRegion, devicePath) + expectLvmInUseDevices = append(expectLvmInUseDevices, devicePath) + } + expectLvmNotInUseDevices = append(expectLvmNotInUseDevices, devicePath) + } + + isVgNeedCreate := true + for _, actualVg := range actualVgConfig { + if expectVgName == actualVg.Name { + isVgNeedCreate = false + if otherUsage := difference(expectLvmInUseDevices, actualVg.PhysicalVolumes); len(otherUsage) != 0 { + log.Errorf("applyRegion:: device [%s] is used in other usage", otherUsage) + break + } + updatePvs := difference(expectLvmNotInUseDevices, actualVg.PhysicalVolumes) + if len(updatePvs) == 0 { + break + } + vrm.updatePmemVg(expectVgName, expectLvmNotInUseDevices) + } + } + if isVgNeedCreate { + if len(expectLvmInUseDevices) != 0 { + log.Errorf("applyRegion:: attempt to use inused devices [%s] to create volumegroup", expectLvmInUseDevices) + continue + } + vrm.createVg(expectVgName, expectLvmNotInUseDevices) + } + } + + return nil +} + +func (vrm *ResourceManager) updatePmemVg(vgName string, addedPv []string) error { + + pvListStr := strings.Join(addedPv, " ") + _, err := vrm.lvmer.ExtendVG(vgName, pvListStr) + if err != nil { + msg := fmt.Sprintf("updatePmemVg:: Extend vg(%s) error: %v", vgName, err) + log.Errorf(msg) + return errors.New(msg) + } + log.Infof("updatePmemVg:: Successful Add pvs %s to VolumeGroup %s", addedPv, vgName) + return nil +} diff --git a/pkg/manager/volumegroup/volumegroup_test.go b/pkg/manager/volumegroup/volumegroup_test.go new file mode 100644 index 0000000..737ab54 --- /dev/null +++ b/pkg/manager/volumegroup/volumegroup_test.go @@ -0,0 +1,200 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumegroup + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/golang/mock/gomock" + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/model" + "github.com/openyurtio/node-resource-manager/pkg/utils" + "github.com/stretchr/testify/assert" + yaml "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func makeValidResourceYaml() *model.ResourceYaml { + return &model.ResourceYaml{ + Name: "foo", + } +} + +func makeResourceYamlCustom(tweaks ...func(*model.ResourceYaml)) *model.ResourceYaml { + resourceYaml := makeValidResourceYaml() + for _, fn := range tweaks { + fn(resourceYaml) + } + return resourceYaml +} + +// EnsureVolumeGroupEnv ... +func EnsureVolumeGroupEnv() (string, error, *ResourceManager) { + config.GlobalConfigVar.NodeInfo = &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{"bar": "foo"}, + }, + } + configPath := "/tmp/volumegroup" + err := EnsureFolder(filepath.Dir(configPath)) + if err != nil { + return "", err, nil + } + + newMockVolumegroupResourceManager := func() *ResourceManager { + return &ResourceManager{ + configPath: configPath, + } + } + return configPath, nil, newMockVolumegroupResourceManager() +} + +func TestAnalyseConfigMap(t *testing.T) { + // set test node info + configPath, err, resourceManager := EnsureVolumeGroupEnv() + if err != nil { + t.Fatal(err) + } + defer os.Remove(configPath) + setOpInOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpIn + m.Value = "foo" + } + setOpNotInOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpNotIn + m.Value = "foo" + } + setOpExistsOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpExists + m.Value = "foo" + } + setOpNotExistsOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpExists + m.Value = "foo" + } + setRyNameBar1Element := func(m *model.ResourceYaml) { + m.Name = "bar1" + } + setDeviceTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "device", + Devices: []string{"/dev/vdb", "/dev/vdc"}, + } + } + setPmemTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "pmem", + Regions: []string{"/dev/vdb", "/dev/vdc"}, + } + } + + testYamls := VgList{VolumeGroups: []model.ResourceYaml{ + *makeResourceYamlCustom(setOpInOperatorElement, setDeviceTopology), + *makeResourceYamlCustom(setOpNotInOperatorElement), + *makeResourceYamlCustom(setOpNotExistsOperatorElement), + *makeResourceYamlCustom(setOpExistsOperatorElement, setRyNameBar1Element, setPmemTopology), + }} + d, err := yaml.Marshal(&testYamls) + defer os.Remove(configPath) + if err != nil { + t.Error() + } + ioutil.WriteFile(configPath, d, 0777) + + assert.Nil(t, resourceManager.AnalyseConfigMap()) + assert.Equal(t, 1, len(resourceManager.volumeGroupDeviceMap)) + assert.Equal(t, 1, len(resourceManager.volumeGroupRegionMap)) +} + +// EnsureFolder ... +func EnsureFolder(target string) error { + mdkirCmd := "mkdir" + _, err := exec.LookPath(mdkirCmd) + if err != nil { + if err == exec.ErrNotFound { + return fmt.Errorf("EnsureFolder:: %q executable not found in $PATH", mdkirCmd) + } + return err + } + + mkdirArgs := []string{"-p", target} + //log.Infof("mkdir for folder, the command is %s %v", mdkirCmd, mkdirArgs) + _, err = exec.Command(mdkirCmd, mkdirArgs...).CombinedOutput() + if err != nil { + return fmt.Errorf("EnsureFolder:: mkdir for folder error: %v", err) + } + return nil +} + +func TestAnalyseDiff(t *testing.T) { + mockCtl := gomock.NewController(t) + defer mockCtl.Finish() + configPath, err, resourceManager := EnsureVolumeGroupEnv() + if err != nil { + t.Fatal(err) + } + defer os.Remove(configPath) + mockPmemer := utils.NewMockPmemer(mockCtl) + resourceManager.pmemer = mockPmemer + mockLVM := utils.NewMockLVM(mockCtl) + resourceManager.lvmer = mockLVM + setOpInOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpIn + m.Value = "foo" + } + setVgDeviceTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "device", + Devices: []string{"/dev/vdb", "/dev/vdc"}, + } + } + setVgDeviceName := func(m *model.ResourceYaml) { + m.Name = "volumegroup1" + } + + testYamls := VgList{VolumeGroups: []model.ResourceYaml{ + *makeResourceYamlCustom(setOpInOperatorElement, setVgDeviceTopology, setVgDeviceName), + }} + d, err := yaml.Marshal(&testYamls) + if err != nil { + t.Error() + } + err = ioutil.WriteFile(configPath, d, 0777) + if err != nil { + t.Fatal(err) + } + prListStr := strings.Join([]string{"/dev/vdb", "/dev/vdc"}, " ") + assert.Nil(t, resourceManager.AnalyseConfigMap()) + gomock.InOrder( + mockLVM.EXPECT().ListPhysicalVolume().Return([]*model.PV{}, nil), + mockLVM.EXPECT().CreateVG(gomock.Eq("volumegroup1"), gomock.Eq(prListStr), gomock.Eq([]string{})).Return("", nil), + ) + assert.Nil(t, resourceManager.ApplyResourceDiff()) +} diff --git a/pkg/model/type.go b/pkg/model/type.go new file mode 100644 index 0000000..ac38f86 --- /dev/null +++ b/pkg/model/type.go @@ -0,0 +1,417 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "fmt" + "strconv" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// VolumeType is volume type +type VolumeType byte + +const ( + // separator for vg info + separator = "<:SEP:>" +) + +var volumeTypeKeys = []byte("mMoOrRsSpviIlcVtTe") + +// types +const ( + VolumeTypeMirrored VolumeType = 'm' + VolumeTypeMirroredWithoutSync VolumeType = 'M' + VolumeTypeOrigin VolumeType = 'o' + VolumeTypeOriginWithMergingSnapshot VolumeType = 'O' + VolumeTypeRAID VolumeType = 'r' + VolumeTypeRAIDWithoutSync VolumeType = 'R' + VolumeTypeSnapshot VolumeType = 's' + VolumeTypeMergingSnapshot VolumeType = 'S' + VolumeTypePVMove VolumeType = 'p' + VolumeTypeVirtualMirror VolumeType = 'v' + VolumeTypeVirtualRaidImage VolumeType = 'i' + VolumeTypeRaidImageOutOfSync VolumeType = 'I' + VolumeTypeMirrorLog VolumeType = 'l' + VolumeTypeUnderConversion VolumeType = 'c' + VolumeTypeThin VolumeType = 'V' + VolumeTypeThinPool VolumeType = 't' + VolumeTypeThinPoolData VolumeType = 'T' + VolumeTypeRaidOrThinPoolMetadata VolumeType = 'e' +) + +// VolumePermissions is volume permissions +type VolumePermissions rune + +var volumePermissonsKeys = []byte("wrR") + +// permissions +const ( + VolumePermissionsWriteable VolumePermissions = 'w' + VolumePermissionsReadOnly VolumePermissions = 'r' + VolumePermissionsReadOnlyActivation VolumePermissions = 'R' +) + +// VolumeAllocation is volume allocation policy +type VolumeAllocation rune + +var volumeAllocationKeys = []byte("acilnACILN") + +// allocations +const ( + VolumeAllocationAnywhere VolumeAllocation = 'a' + VolumeAllocationContiguous VolumeAllocation = 'c' + VolumeAllocationInherited VolumeAllocation = 'i' + VolumeAllocationCling VolumeAllocation = 'l' + VolumeAllocationNormal VolumeAllocation = 'n' + VolumeAllocationAnywhereLocked VolumeAllocation = 'A' + VolumeAllocationContiguousLocked VolumeAllocation = 'C' + VolumeAllocationInheritedLocked VolumeAllocation = 'I' + VolumeAllocationClingLocked VolumeAllocation = 'L' + VolumeAllocationNormalLocked VolumeAllocation = 'N' +) + +// VolumeFixedMinor is volume fixed minor +type VolumeFixedMinor rune + +// fixed minor +const ( + VolumeFixedMinorEnabled VolumeFixedMinor = 'm' + VolumeFixedMinorDisabled VolumeFixedMinor = '-' +) + +func (t VolumeFixedMinor) toProto() bool { + return t == VolumeFixedMinorEnabled +} + +// VolumeState is volume state +type VolumeState rune + +var volumeStateKeys = []byte("asISmMdi") + +// states +const ( + VolumeStateActive VolumeState = 'a' + VolumeStateSuspended VolumeState = 's' + VolumeStateInvalidSnapshot VolumeState = 'I' + VolumeStateInvalidSuspendedSnapshot VolumeState = 'S' + VolumeStateSnapshotMergeFailed VolumeState = 'm' + VolumeStateSuspendedSnapshotMergeFailed VolumeState = 'M' + VolumeStateMappedDevicePresentWithoutTables VolumeState = 'd' + VolumeStateMappedDevicePresentWithInactiveTable VolumeState = 'i' +) + +// VolumeOpen is volume open +type VolumeOpen rune + +// open +const ( + VolumeOpenIsOpen VolumeOpen = 'o' + VolumeOpenIsNotOpen VolumeOpen = '-' +) + +// VolumeTargetType is volume taget type +type VolumeTargetType rune + +var volumeTargetTypeKeys = []byte("mrstuv") + +// target type +const ( + VolumeTargetTypeMirror VolumeTargetType = 'm' + VolumeTargetTypeRAID VolumeTargetType = 'r' + VolumeTargetTypeSnapshot VolumeTargetType = 's' + VolumeTargetTypeThin VolumeTargetType = 't' + VolumeTargetTypeUnknown VolumeTargetType = 'u' + VolumeTargetTypeVirtual VolumeTargetType = 'v' +) + +// VolumeZeroing is volume zeroing +type VolumeZeroing rune + +// zeroing +const ( + VolumeZeroingIsZeroing VolumeZeroing = 'z' + VolumeZeroingIsNonZeroing VolumeZeroing = '-' +) + +// VolumeHealth is volume health +type VolumeHealth rune + +// health +const ( + VolumeHealthOK VolumeHealth = '-' + VolumeHealthPartial VolumeHealth = 'p' + VolumeHealthRefreshNeeded VolumeHealth = 'r' + VolumeHealthMismatchesExist VolumeHealth = 'm' + VolumeHealthWritemostly VolumeHealth = 'w' +) + +// VolumeActivationSkipped is volume activation +type VolumeActivationSkipped rune + +// activation +const ( + VolumeActivationSkippedIsSkipped VolumeActivationSkipped = 's' + VolumeActivationSkippedIsNotSkipped VolumeActivationSkipped = '-' +) + +func (t VolumeActivationSkipped) toProto() bool { + return t == VolumeActivationSkippedIsSkipped +} + +// ResourceYaml ... +type ResourceYaml struct { + Name string `yaml:"name,omitempty"` + Key string `yaml:"key,omitempty"` + Operator metav1.LabelSelectorOperator `yaml:"operator,omitempty"` + Value string `yaml:"value,omitempty"` + Topology Topology `yaml:"topology,omitempty"` +} + +// Topology ... +type Topology struct { + Type string `yaml:"type,omitempty"` + Options string `yaml:"options,omitempty"` + Fstype string `yaml:"fstype,omitempty"` + + Devices []string `yaml:"devices,omitempty"` + Volumes []map[string]string `yaml:"volumes,omitempty"` + Regions []string `yaml:"regions,omitempty"` +} + +// PmemRegions list all regions +type PmemRegions struct { + Regions []PmemRegion `json:"regions"` +} + +// PmemRegion define on pmem region +type PmemRegion struct { + Dev string `json:"dev"` + Size int64 `json:"size,omitempty"` + AvailableSize int64 `json:"available_size,omitempty"` + MaxAvailableExent int64 `json:"max_available_extent,omitempty"` + RegionType string `json:"type,omitempty"` + IsetID int64 `json:"iset_id,omitempty"` + PersistenceDomain string `json:"persistence_domain,omitempty"` + Namespaces []PmemNameSpace `json:"namespaces,omitempty"` +} + +// PmemNameSpace define one pmem namespaces +type PmemNameSpace struct { + Dev string `json:"dev,omitempty"` + Mode string `json:"mode,omitempty"` + MapType string `json:"map,omitempty"` + Size int64 `json:"size,omitempty"` + UUID string `json:"uuid,omitempty"` + SectorSize int64 `json:"sectorsize,omitempty"` + Align int64 `json:"align,omitempty"` + BlockDev string `json:"blockdev,omitempty"` + CharDev string `json:"chardev,omitempty"` + Name string `json:"name,omitempty"` +} + +// DaxctrlMem list all mems +type DaxctrlMem struct { + Chardev string `json:"chardev"` + Size int64 `json:"size"` + TargetNode int `json:"target_node"` + Mode string `json:"mode"` + Movable bool `json:"movable"` +} + +// LV is a logical volume +type LV struct { + Name string + Size uint64 + UUID string + Attributes LVAttributes + CopyPercent string + ActualDevMajNumber uint32 + ActualDevMinNumber uint32 + Tags []string +} + +// VG is volume group +type VG struct { + Name string + Size uint64 + FreeSize uint64 + UUID string + Tags []string +} + +// PV is Physica lVolume +type PV struct { + Name string + VgName string + Size uint64 + UUID string +} + +// LVAttributes is attributes +type LVAttributes struct { + Type VolumeType + Permissions VolumePermissions + Allocation VolumeAllocation + FixedMinor VolumeFixedMinor + State VolumeState + Open VolumeOpen + TargetType VolumeTargetType + Zeroing VolumeZeroing + Health VolumeHealth + ActivationSkipped VolumeActivationSkipped +} + +// ParseLV ... +func ParseLV(line string) (*LV, error) { + // lvs --units=b --separator="<:SEP:>" --nosuffix --noheadings -o lv_name,lv_size,lv_uuid,lv_attr,copy_percent,lv_kernel_major,lv_kernel_minor,lv_tags --nameprefixes -a + // todo: devices, lv_ancestors, lv_descendants, lv_major, lv_minor, mirror_log, modules, move_pv, origin, region_size + // seg_count, seg_size, seg_start, seg_tags, segtype, snap_percent, stripes, stripe_size + fields, err := parse(line, 8) + if err != nil { + return nil, err + } + + size, err := strconv.ParseUint(fields["LVM2_LV_SIZE"], 10, 64) + if err != nil { + return nil, err + } + + kernelMajNumber, err := strconv.ParseUint(fields["LVM2_LV_KERNEL_MAJOR"], 10, 32) + if err != nil { + return nil, err + } + + kernelMinNumber, err := strconv.ParseUint(fields["LVM2_LV_KERNEL_MINOR"], 10, 32) + if err != nil { + return nil, err + } + + attrs, err := parseAttrs(fields["LVM2_LV_ATTR"]) + if err != nil { + return nil, err + } + + return &LV{ + Name: fields["LVM2_LV_NAME"], + Size: size, + UUID: fields["LVM2_LV_UUID"], + Attributes: *attrs, + CopyPercent: fields["LVM2_COPY_PERCENT"], + ActualDevMajNumber: uint32(kernelMajNumber), + ActualDevMinNumber: uint32(kernelMinNumber), + Tags: strings.Split(fields["LVM2_LV_TAGS"], ","), + }, nil + +} + +// ParseVG parse volume group +func ParseVG(line string) (*VG, error) { + // vgs --units=b --separator="<:SEP:>" --nosuffix --noheadings -o vg_name,vg_size,vg_free,vg_uuid,vg_tags --nameprefixes -a + fields, err := parse(line, 5) + if err != nil { + return nil, err + } + + size, err := strconv.ParseUint(fields["LVM2_VG_SIZE"], 10, 64) + if err != nil { + return nil, err + } + + freeSize, err := strconv.ParseUint(fields["LVM2_VG_FREE"], 10, 64) + if err != nil { + return nil, err + } + + return &VG{ + Name: fields["LVM2_VG_NAME"], + Size: size, + FreeSize: freeSize, + UUID: fields["LVM2_VG_UUID"], + Tags: strings.Split(fields["LVM2_VG_TAGS"], ","), + }, nil +} + +// ParsePV parse volume group +func ParsePV(line string) (*PV, error) { + // vgs --units=b --separator="<:SEP:>" --nosuffix --noheadings -o vg_name,vg_size,vg_free,vg_uuid,vg_tags --nameprefixes -a + fields, err := parse(line, 4) + if err != nil { + return nil, err + } + + size, err := strconv.ParseUint(fields["LVM2_PV_SIZE"], 10, 64) + if err != nil { + return nil, err + } + + return &PV{ + Name: fields["LVM2_PV_NAME"], + VgName: fields["LVM2_VG_NAME"], + Size: size, + UUID: fields["LVM2_PV_UUID"], + }, nil +} + +func parse(line string, numComponents int) (map[string]string, error) { + components := strings.Split(line, separator) + if len(components) != numComponents { + return nil, fmt.Errorf("expected %d components, got %d", numComponents, len(components)) + } + + fields := map[string]string{} + for _, c := range components { + idx := strings.Index(c, "=") + if idx == -1 { + return nil, fmt.Errorf("failed to parse component '%s'", c) + } + key := c[0:idx] + value := c[idx+1:] + if len(value) < 2 { + return nil, fmt.Errorf("failed to parse component '%s'", c) + } + if value[0] != '\'' || value[len(value)-1] != '\'' { + return nil, fmt.Errorf("failed to parse component '%s'", c) + } + value = value[1 : len(value)-1] + fields[key] = value + } + + return fields, nil +} + +func parseAttrs(attrs string) (*LVAttributes, error) { + if len(attrs) != 10 { + return nil, fmt.Errorf("incorrect attrs block size, expected 10, got %d in %s", len(attrs), attrs) + } + + ret := &LVAttributes{} + ret.Type = VolumeType(attrs[0]) + ret.Permissions = VolumePermissions(attrs[1]) + ret.Allocation = VolumeAllocation(attrs[2]) + ret.FixedMinor = VolumeFixedMinor(attrs[3]) + ret.State = VolumeState(attrs[4]) + ret.Open = VolumeOpen(attrs[5]) + ret.TargetType = VolumeTargetType(attrs[6]) + ret.Zeroing = VolumeZeroing(attrs[7]) + ret.Health = VolumeHealth(attrs[8]) + ret.ActivationSkipped = VolumeActivationSkipped(attrs[9]) + + return ret, nil +} diff --git a/pkg/signals/signal_posix.go b/pkg/signals/signal_posix.go new file mode 100644 index 0000000..1bafca8 --- /dev/null +++ b/pkg/signals/signal_posix.go @@ -0,0 +1,25 @@ +// +build !windows + +/* +Copyright 2021 The OpenYurt Authors. +Copyright 2018 The Knative Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package signals + +import ( + "os" + "syscall" +) + +var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} diff --git a/pkg/signals/signals.go b/pkg/signals/signals.go new file mode 100644 index 0000000..3e52837 --- /dev/null +++ b/pkg/signals/signals.go @@ -0,0 +1,82 @@ +/* +Copyright 2021 The OpenYurt Authors. +Copyright 2018 The Knative Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package signals + +import ( + "context" + "errors" + "os" + "os/signal" + "time" +) + +var onlyOneSignalHandler = make(chan struct{}) + +// SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned +// which is closed on one of these signals. If a second signal is caught, the program +// is terminated with exit code 1. +func SetupSignalHandler() (stopCh <-chan struct{}) { + close(onlyOneSignalHandler) // panics when called twice + + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + <-c + close(stop) + <-c + os.Exit(1) // second signal. Exit directly. + }() + + return stop +} + +// NewContext creates a new context with SetupSignalHandler() +// as our Done() channel. +func NewContext() context.Context { + return &signalContext{stopCh: SetupSignalHandler()} +} + +type signalContext struct { + stopCh <-chan struct{} +} + +// Deadline implements context.Context +func (scc *signalContext) Deadline() (deadline time.Time, ok bool) { + return +} + +// Done implements context.Context +func (scc *signalContext) Done() <-chan struct{} { + return scc.stopCh +} + +// Err implements context.Context +func (scc *signalContext) Err() error { + select { + case _, ok := <-scc.Done(): + if !ok { + return errors.New("received a termination signal") + } + default: + } + return nil +} + +// Value implements context.Context +func (scc *signalContext) Value(key interface{}) interface{} { + return nil +} diff --git a/pkg/utils/lvm.go b/pkg/utils/lvm.go new file mode 100644 index 0000000..72aa83c --- /dev/null +++ b/pkg/utils/lvm.go @@ -0,0 +1,297 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "errors" + "fmt" + "strings" + + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +const ( + ProtectedTagName = "protected" +) + +// LVM ... +type LVM interface { + ListLV(listSpec string) ([]*model.LV, error) + CreateLV(vg, name string, size uint64, mirrors uint32, tags []string) (string, error) + RemoveLV(vg, name string) (string, error) + CloneLV(src, dest string) (string, error) + ListVG() ([]*model.VG, error) + ListPhysicalVolume() ([]*model.PV, error) + CreateVG(name, physicalVolume string, tags []string) (string, error) + ExtendVG(name, physicalVolume string) (string, error) + RemoveVG(name string) (string, error) + AddTagLV(vg, name string, tags []string) (string, error) + RemoveTagLV(vg, name string, tags []string) (string, error) +} + +// NodeLVM ... +type NodeLVM struct { +} + +// NewNodeLVM ... +func NewNodeLVM() *NodeLVM { + return &NodeLVM{} +} + +// ListLV ... +func (nl *NodeLVM) ListLV(listSpec string) ([]*model.LV, error) { + cmdList := []string{NsenterCmd, "lvs", "--units=b", "--separator=\"<:SEP:>\"", "--nosuffix", "--noheadings", + "-o", "lv_name,lv_size,lv_uuid,lv_attr,copy_percent,lv_kernel_major,lv_kernel_minor,lv_tags", "--nameprefixes", "-a", listSpec} + cmd := strings.Join(cmdList, " ") + out, err := Run(cmd) + if err != nil { + return nil, err + } + outStr := strings.TrimSpace(string(out)) + if outStr == "" { + lvs := make([]*model.LV, 0) + return lvs, nil + } + outLines := strings.Split(outStr, "\n") + lvs := []*model.LV{} + for _, line := range outLines { + line = strings.TrimSpace(line) + if !strings.Contains(line, "LVM2_LV_NAME") { + continue + } + lv, err := model.ParseLV(line) + if err != nil { + return nil, errors.New("Parse LVM: " + line + ", with error: " + err.Error()) + } + lvs = append(lvs, lv) + } + return lvs, nil +} + +// CreateLV ... +func (nl *NodeLVM) CreateLV(vg, name string, size uint64, mirrors uint32, tags []string) (string, error) { + if size == 0 { + return "", errors.New("size must be greater than 0") + } + + args := []string{"lvcreate", "-v", "-n", name, "-L", fmt.Sprintf("%db", size)} + if mirrors > 0 { + args = append(args, "-m", fmt.Sprintf("%d", mirrors), "--nosync") + } + for _, tag := range tags { + args = append(args, "--add-tag", tag) + } + + args = append(args, vg) + cmd := strings.Join(args, " ") + out, err := Run(cmd) + return string(out), err +} + +// RemoveLV ... +func (nl *NodeLVM) RemoveLV(vg, name string) (string, error) { + lvs, err := nl.ListLV(fmt.Sprintf("%s/%s", vg, name)) + if err != nil { + return "", fmt.Errorf("failed to list LVs: %v", err) + } + if len(lvs) == 0 { + return "lvm " + vg + "/" + name + " is not exist, skip remove", nil + } + if len(lvs) != 1 { + return "", fmt.Errorf("expected 1 LV, got %d", len(lvs)) + } + for _, tag := range lvs[0].Tags { + if tag == ProtectedTagName { + return "", errors.New("volume is protected") + } + } + + args := []string{NsenterCmd, "lvremove", "-v", "-f", fmt.Sprintf("%s/%s", vg, name)} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err + +} + +// CloneLV ... +func (nl *NodeLVM) CloneLV(src, dest string) (string, error) { + args := []string{NsenterCmd, "dd", fmt.Sprintf("if=%s", src), fmt.Sprintf("of=%s", dest), "bs=4M"} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err +} + +// ListVG ... +func (nl *NodeLVM) ListVG() ([]*model.VG, error) { + args := []string{NsenterCmd, "vgs", "--units=b", "--separator=\"<:SEP:>\"", "--nosuffix", "--noheadings", + "-o", "vg_name,vg_size,vg_free,vg_uuid,vg_tags", "--nameprefixes", "-a"} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + if err != nil { + return nil, err + } + vgs := []*model.VG{} + outStr := strings.TrimSpace(string(out)) + if outStr == "" { + return vgs, nil + } + outLines := strings.Split(outStr, "\n") + for _, line := range outLines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "WARNING") { + continue + } + vg, err := model.ParseVG(line) + if err != nil { + return nil, err + } + vgs = append(vgs, vg) + } + return vgs, nil +} + +// ListPhysicalVolume ... +func (nl *NodeLVM) ListPhysicalVolume() ([]*model.PV, error) { + args := []string{NsenterCmd, "pvs", "--units=b", "--separator=\"<:SEP:>\"", "--nosuffix", "--noheadings", + "-o", "vg_name,pv_name,pv_size,pv_uuid", "--nameprefixes", "-a"} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + if err != nil { + return nil, err + } + outStr := strings.TrimSpace(string(out)) + pvs := []*model.PV{} + if outStr == "" { + return pvs, nil + } + outLines := strings.Split(outStr, "\n") + for _, line := range outLines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "WARNING") { + continue + } + pv, err := model.ParsePV(line) + if err != nil { + return nil, err + } + if pv.VgName != "" && pv.Name != "" { + pvs = append(pvs, pv) + } + } + return pvs, nil +} + +// CreateVG ... +func (nl *NodeLVM) CreateVG(name, physicalVolume string, tags []string) (string, error) { + args := []string{NsenterCmd, "vgcreate", name, physicalVolume, "-v"} + for _, tag := range tags { + args = append(args, "--add-tag", tag) + } + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err +} + +// ExtendVG ... +func (nl *NodeLVM) ExtendVG(name, physicalVolume string) (string, error) { + args := []string{NsenterCmd, "vgextend", name, physicalVolume, "-v"} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err +} + +// RemoveVG ... +func (nl *NodeLVM) RemoveVG(name string) (string, error) { + vgs, err := nl.ListVG() + if err != nil { + return "", fmt.Errorf("failed to list VGs: %v", err) + } + var vg *model.VG + for _, v := range vgs { + if v.Name == name { + vg = v + break + } + } + if vg == nil { + return "", fmt.Errorf("could not find vg to delete") + } + for _, tag := range vg.Tags { + if tag == ProtectedTagName { + return "", errors.New("volume is protected") + } + } + + args := []string{NsenterCmd, "vgremove", "-v", "-f", name} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err + +} + +// AddTagLV ... +func (nl *NodeLVM) AddTagLV(vg, name string, tags []string) (string, error) { + lvs, err := nl.ListLV(fmt.Sprintf("%s/%s", vg, name)) + if err != nil { + return "", fmt.Errorf("failed to list LVs: %v", err) + } + if len(lvs) != 1 { + return "", fmt.Errorf("expected 1 LV, got %d", len(lvs)) + } + + args := make([]string, 0) + args = append(args, NsenterCmd) + args = append(args, "lvchange") + for _, tag := range tags { + args = append(args, "--addtag", tag) + } + + args = append(args, fmt.Sprintf("%s/%s", vg, name)) + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err +} + +// RemoveTagLV .... +func (nl *NodeLVM) RemoveTagLV(vg, name string, tags []string) (string, error) { + + lvs, err := nl.ListLV(fmt.Sprintf("%s/%s", vg, name)) + if err != nil { + return "", fmt.Errorf("failed to list LVs: %v", err) + } + if len(lvs) != 1 { + return "", fmt.Errorf("expected 1 LV, got %d", len(lvs)) + } + + args := make([]string, 0) + args = append(args, NsenterCmd) + args = append(args, "lvchange") + for _, tag := range tags { + args = append(args, "--deltag", tag) + } + + args = append(args, fmt.Sprintf("%s/%s", vg, name)) + cmd := strings.Join(args, " ") + out, err := Run(cmd) + return string(out), err +} diff --git a/pkg/utils/lvm_mock.go b/pkg/utils/lvm_mock.go new file mode 100644 index 0000000..03b1aba --- /dev/null +++ b/pkg/utils/lvm_mock.go @@ -0,0 +1,214 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "reflect" + + "github.com/golang/mock/gomock" + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +// MockLVM ... +type MockLVM struct { + ctrl *gomock.Controller + recorder *MockLVMMockRecorder +} + +// MockLVMMockRecorder ... +type MockLVMMockRecorder struct { + mock *MockLVM +} + +// EXPECT ... +func (m *MockLVM) EXPECT() *MockLVMMockRecorder { + return m.recorder +} + +// NewMockLVM ... +func NewMockLVM(ctrl *gomock.Controller) *MockLVM { + mock := &MockLVM{ctrl: ctrl} + mock.recorder = &MockLVMMockRecorder{mock} + return mock +} + +// ListLV ... +func (m *MockLVM) ListLV(listSpec string) ([]*model.LV, error) { + + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListLV", listSpec) + ret0, _ := ret[0].([]*model.LV) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListLV ... +func (mr *MockLVMMockRecorder) ListLV(arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLV", reflect.TypeOf((*MockLVM)(nil).ListLV), arg1) +} + +// CreateLV ... +func (m *MockLVM) CreateLV(vg, name string, size uint64, mirrors uint32, tags []string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateLV", vg, name, size, mirrors, tags) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateLV ... +func (mr *MockLVMMockRecorder) CreateLV(arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateLV", reflect.TypeOf((*MockLVM)(nil).CreateLV), arg1, arg2, arg3, arg4, arg5) +} + +// RemoveLV ... +func (m *MockLVM) RemoveLV(vg, name string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveLV", vg, name) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveLV ... +func (mr *MockLVMMockRecorder) RemoveLV(arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveLV", reflect.TypeOf((*MockLVM)(nil).RemoveLV), arg1, arg2) +} + +// CloneLV ... +func (m *MockLVM) CloneLV(src, dest string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloneLV", src, dest) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CloneLV ... +func (mr *MockLVMMockRecorder) CloneLV(arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloneLV", reflect.TypeOf((*MockLVM)(nil).CloneLV), arg1, arg2) +} + +// ListVG ... +func (m *MockLVM) ListVG() ([]*model.VG, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListVG") + ret0, _ := ret[0].([]*model.VG) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListVG ... +func (mr *MockLVMMockRecorder) ListVG() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListVG", reflect.TypeOf((*MockLVM)(nil).ListVG)) +} + +// ListPhysicalVolume ... +func (m *MockLVM) ListPhysicalVolume() ([]*model.PV, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListPhysicalVolume") + ret0, _ := ret[0].([]*model.PV) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListPhysicalVolume ... +func (mr *MockLVMMockRecorder) ListPhysicalVolume() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPhysicalVolume", reflect.TypeOf((*MockLVM)(nil).ListPhysicalVolume)) +} + +// CreateVG ... +func (m *MockLVM) CreateVG(name, physicalVolume string, tags []string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateVG", name, physicalVolume, tags) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 + +} + +// CreateVG ... +func (mr *MockLVMMockRecorder) CreateVG(arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateVG", reflect.TypeOf((*MockLVM)(nil).CreateVG), arg1, arg2, arg3) +} + +// ExtendVG ... +func (m *MockLVM) ExtendVG(name, physicalVolume string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExtendVG", name, physicalVolume) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ExtendVG ... +func (mr *MockLVMMockRecorder) ExtendVG(arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExtendVG", reflect.TypeOf((*MockLVM)(nil).ExtendVG), arg1, arg2) +} + +// RemoveVG ... +func (m *MockLVM) RemoveVG(name string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExtendVG", name) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveVG ... +func (mr *MockLVMMockRecorder) RemoveVG(arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveVG", reflect.TypeOf((*MockLVM)(nil).RemoveVG), arg1) +} + +// AddTagLV ... +func (m *MockLVM) AddTagLV(vg, name string, tags []string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddTagLV", vg, name, tags) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddTagLV ... +func (mr *MockLVMMockRecorder) AddTagLV(arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddTagLV", reflect.TypeOf((*MockLVM)(nil).AddTagLV), arg1, arg2, arg3) +} + +// RemoveTagLV ... +func (m *MockLVM) RemoveTagLV(vg, name string, tags []string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveTagLV", vg, name, tags) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveTagLV ... +func (mr *MockLVMMockRecorder) RemoveTagLV(arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveTagLV", reflect.TypeOf((*MockLVM)(nil).RemoveTagLV), arg1, arg2, arg3) +} diff --git a/pkg/utils/mounter.go b/pkg/utils/mounter.go new file mode 100644 index 0000000..56f2110 --- /dev/null +++ b/pkg/utils/mounter.go @@ -0,0 +1,263 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + + log "github.com/sirupsen/logrus" + utilexec "k8s.io/utils/exec" + k8smount "k8s.io/utils/mount" +) + +const ( + // fsckErrorsCorrected tag + fsckErrorsCorrected = 1 + // fsckErrorsUncorrected tag + fsckErrorsUncorrected = 4 +) + +type findmntResponse struct { + FileSystems []fileSystem `json:"filesystems"` +} + +type fileSystem struct { + Target string `json:"target"` + Propagation string `json:"propagation"` + FsType string `json:"fstype"` + Options string `json:"options"` +} + +// Mounter is responsible for formatting and mounting volumes +type Mounter interface { + k8smount.Interface + utilexec.Interface + // If the folder doesn't exist, it will call 'mkdir -p' + EnsureFolder(string) error + // FormatAndMount ... + FormatAndMount(string, string, string, []string, string) error + + // IsMounted checks whether the target path is a correct mount (i.e: + // propagated). It returns true if it's mounted. An error is returned in + // case of system errors or if it's mounted incorrectly. + IsMounted(target string) (bool, error) + + SafePathRemove(target string) error +} + +// TODO(arslan): this is Linux only for now. Refactor this into a package with +// architecture specific code in the future, such as mounter_darwin.go, +// mounter_linux.go, etc.. +type NodeMounter struct { + k8smount.SafeFormatAndMount + utilexec.Interface +} + +// NewMounter returns a new mounter instance +func NewMounter() Mounter { + return &NodeMounter{ + k8smount.SafeFormatAndMount{ + Interface: k8smount.New(""), + Exec: utilexec.New(), + }, + utilexec.New(), + } +} + +// EnsureFolder ... +func (m *NodeMounter) EnsureFolder(target string) error { + mkdirCmd := "mkdir" + _, err := m.LookPath(mkdirCmd) + if err != nil { + if err == exec.ErrNotFound { + return fmt.Errorf("%q executable not found in $PATH", mkdirCmd) + } + return err + } + + mkdirCmd = NsenterCmd + mkdirCmd + mkdirCmd += fmt.Sprintf(" -p %s", target) + log.Infof("mkdir for folder, the command is %s", mkdirCmd) + output, err := Run(mkdirCmd) + if err != nil { + return fmt.Errorf("EnsureFolder:: mkdir for folder output: %s error: %v", output, err) + } + return nil +} + +// FormatAndMount ... +func (m *NodeMounter) FormatAndMount(source, target, fstype string, mkfsOptions []string, mountOptions string) error { + // diskMounter.Interface = m.K8smounter + readOnly := false + + if !readOnly { + // Run fsck on the disk to fix repairable issues, only do this for volumes requested as rw. + args := []string{"-a", source} + + out, err := m.Exec.Command("fsck", args...).CombinedOutput() + if err != nil { + ee, isExitError := err.(utilexec.ExitError) + switch { + case err == utilexec.ErrExecutableNotFound: + log.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.") + case isExitError && ee.ExitStatus() == fsckErrorsCorrected: + log.Infof("Device %s has errors which were corrected by fsck.", source) + case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: + return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s", source, string(out)) + case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: + } + } + } + + // Try to mount the disk + cmd := fmt.Sprintf("%smount -o %s %s %s", NsenterCmd, mountOptions, source, target) + log.Infof("FormatAndMount:: cmd: %s", cmd) + output, mountErr := Run(cmd) + if mountErr != nil { + // Mount failed. This indicates either that the disk is unformatted or + // it contains an unexpected filesystem. + existingFormat, err := m.GetDiskFormat(source) + + if err != nil { + return err + } + if existingFormat == "" { + if readOnly { + // Don't attempt to format if mounting as readonly, return an error to reflect this. + return errors.New("failed to mount unformatted volume as read only") + } + + // Disk is unformatted so format it. + args := []string{source} + // Use 'ext4' as the default + if len(fstype) == 0 { + fstype = "ext4" + } + + if fstype == "ext4" || fstype == "ext3" { + args = []string{ + "-F", // Force flag + "-m0", // Zero blocks reserved for super-user + source, + } + // add mkfs options + if len(mkfsOptions) != 0 { + args = []string{} + for _, opts := range mkfsOptions { + args = append(args, opts) + } + args = append(args, source) + } + } + log.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) + + mkfsCmd := fmt.Sprintf("%s mkfs.%s %s", NsenterCmd, fstype, strings.Join(args, " ")) + log.Infof("FormatAndMount:: mkfscmd: %s", mkfsCmd) + _, err = Run(mkfsCmd) + if err == nil { + // the disk has been formatted successfully try to mount it again. + output, mountErr := Run(cmd) + log.Infof("FormatAndMount:: cmd output %s", output) + return mountErr + } + log.Errorf("format of disk %q failed: type:(%q) target:(%q) options:(%q) output: (%s) error:(%v)", source, fstype, target, mkfsOptions, output, err) + return err + } + // Disk is already formatted and failed to mount + if len(fstype) == 0 || fstype == existingFormat { + // This is mount error + return mountErr + } + // Block device is formatted with unexpected filesystem, let the user know + return fmt.Errorf("failed to mount the volume as %q, it already contains %s. Mount error: %v", fstype, existingFormat, mountErr) + } + + return mountErr +} + +// IsMounted ... +func (m *NodeMounter) IsMounted(target string) (bool, error) { + if target == "" { + return false, errors.New("target is not specified for checking the mount") + } + findmntCmd := "grep" + findmntArgs := []string{target, "/proc/mounts"} + out, err := exec.Command(findmntCmd, findmntArgs...).CombinedOutput() + outStr := strings.TrimSpace(string(out)) + if err != nil { + if outStr == "" { + return false, nil + } + return false, fmt.Errorf("checking mounted failed: %v cmd: %q output: %q", + err, findmntCmd, outStr) + } + if strings.Contains(outStr, target) { + return true, nil + } + return false, nil +} + +// SafePathRemove ... +func (m *NodeMounter) SafePathRemove(targetPath string) error { + fo, err := os.Lstat(targetPath) + if err != nil { + return err + } + isMounted, err := m.IsMounted(targetPath) + if err != nil { + return err + } + if isMounted { + return errors.New("Path is mounted, not remove: " + targetPath) + } + if fo.IsDir() { + empty, err := IsDirEmpty(targetPath) + if err != nil { + return errors.New("Check path empty error: " + targetPath + err.Error()) + } + if !empty { + return errors.New("Cannot remove Path not empty: " + targetPath) + } + } + err = os.Remove(targetPath) + if err != nil { + return err + } + return nil +} + +// IsDirEmpty return status of dir empty or not +func IsDirEmpty(name string) (bool, error) { + f, err := os.Open(name) + if err != nil { + return false, err + } + defer f.Close() + + // read in ONLY one file + _, err = f.Readdir(1) + // and if the file is EOF... well, the dir is empty. + if err == io.EOF { + return true, nil + } + return false, err +} diff --git a/pkg/utils/mounter_mock.go b/pkg/utils/mounter_mock.go new file mode 100644 index 0000000..9f67cf6 --- /dev/null +++ b/pkg/utils/mounter_mock.go @@ -0,0 +1,107 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "reflect" + + "github.com/golang/mock/gomock" + utilexec "k8s.io/utils/exec" + k8smount "k8s.io/utils/mount" +) + +// MockMounter ... +type MockMounter struct { + k8smount.SafeFormatAndMount + utilexec.Interface + ctrl *gomock.Controller + recorder *MockMounterMockRecorder +} + +// MockMounterMockRecorder ... +type MockMounterMockRecorder struct { + mock *MockMounter +} + +// NewMockMounter ... +func NewMockMounter(ctrl *gomock.Controller) *MockMounter { + mock := &MockMounter{ctrl: ctrl} + mock.recorder = &MockMounterMockRecorder{mock} + return mock +} + +// EXPECT ... +func (m *MockMounter) EXPECT() *MockMounterMockRecorder { + return m.recorder +} + +// EnsureFolder ... +func (m *MockMounter) EnsureFolder(target string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureFolder", target) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureFolder ... +func (mr MockMounterMockRecorder) EnsureFolder(target interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureFolder", reflect.TypeOf((*MockMounter)(nil).EnsureFolder), target) +} + +// FormatAndMount ... +func (m *MockMounter) FormatAndMount(source, target, fstype string, mkfsOptions []string, mountOptions string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FormatAndMount", source, target, fstype, mkfsOptions, mountOptions) + ret0, _ := ret[0].(error) + return ret0 +} + +// FormatAndMount ... +func (mr MockMounterMockRecorder) FormatAndMount(arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FormatAndMount", reflect.TypeOf((*MockMounter)(nil).FormatAndMount), arg1, arg2, arg3, arg4, arg5) +} + +// IsMounted ... +func (m *MockMounter) IsMounted(target string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsMounted", target) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsMounted ... +func (mr MockMounterMockRecorder) IsMounted(target interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsMounted", reflect.TypeOf((*MockMounter)(nil).IsMounted), target) +} + +// SafePathRemove ... +func (m MockMounter) SafePathRemove(targetPath string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SafePathRemove", targetPath) + ret0, _ := ret[0].(error) + return ret0 +} + +// SafePathRemove ... +func (mr MockMounterMockRecorder) SafePathRemove(target interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SafePathRemove", reflect.TypeOf((*MockMounter)(nil).SafePathRemove), target) +} diff --git a/pkg/utils/pmem.go b/pkg/utils/pmem.go new file mode 100644 index 0000000..af7d6b0 --- /dev/null +++ b/pkg/utils/pmem.go @@ -0,0 +1,173 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/openyurtio/node-resource-manager/pkg/model" + log "github.com/sirupsen/logrus" +) + +// Pmemer ... +type Pmemer interface { + GetRegions() (*model.PmemRegions, error) + CreateNamespace(string, string) error + CheckNamespaceUsed(string) bool + GetPmemNamespaceDeivcePath(string, string) (string, string, error) + MakeNamespaceMemory(chardev string) error + CheckKMEMCreated(chardev string) (bool, error) +} + +// NodePmemer ... +type NodePmemer struct { +} + +// NewNodePmemer create new NodePmemer struct +func NewNodePmemer() *NodePmemer { + return &NodePmemer{} +} + +// GetRegions ... +func (np *NodePmemer) GetRegions() (*model.PmemRegions, error) { + regions := &model.PmemRegions{} + getRegionCmd := fmt.Sprintf("%s ndctl list -RN", NsenterCmd) + regionOut, err := Run(getRegionCmd) + if err != nil { + return regions, err + } + err = json.Unmarshal(([]byte)(regionOut), regions) + if err != nil { + if strings.HasPrefix(regionOut, "[") { + regionList := []model.PmemRegion{} + err = json.Unmarshal(([]byte)(regionOut), ®ionList) + if err != nil { + return regions, err + } + regions.Regions = regionList + } else { + return regions, err + } + } + return regions, nil +} + +// CreateNamespace ... +func (np *NodePmemer) CreateNamespace(region, pmemType string) error { + var createCmd string + if pmemType == "lvm" { + createCmd = fmt.Sprintf("%s ndctl create-namespace -r %s", NsenterCmd, region) + } else { + createCmd = fmt.Sprintf("%s ndctl create-namespace -r %s --mode=devdax", NsenterCmd, region) + } + _, err := Run(createCmd) + if err != nil { + log.Errorf("Create NameSpace for region %s error: %v", region, err) + return err + } + log.Infof("Create NameSpace for region %s successful", region) + return nil +} + +// CheckNamespaceUsed device used in block +func (np *NodePmemer) CheckNamespaceUsed(devicePath string) bool { + pvCheckCmd := fmt.Sprintf("%s pvs %s 2>&1 | grep -v \"Failed to \" | grep /dev | awk '{print $2}' | wc -l", NsenterCmd, devicePath) + out, err := Run(pvCheckCmd) + if err == nil && strings.TrimSpace(out) != "0" { + log.Infof("CheckNamespaceUsed: NameSpace %s used for pv", devicePath) + return true + } + + out, err = checkFSType(devicePath) + if err == nil && strings.TrimSpace(out) != "" { + log.Infof("CheckNamespaceUsed: NameSpace %s format as %s", devicePath, out) + return true + } + return false +} + +// GetPmemNamespaceDeivcePath ... +func (np *NodePmemer) GetPmemNamespaceDeivcePath(region, mode string) (devicePath string, namespaceName string, err error) { + regions, err := np.getRegionNamespaceInfo(region) + if err != nil { + return "", "", err + } + namespace := regions.Regions[0].Namespaces[0] + if namespace.Mode != mode { + log.Errorf("GetPmemNamespaceDeivcePath namespace mode %s wrong with: %s", namespace.Mode, mode) + return "", "", errors.New("GetPmemNamespaceDeivcePath pmem namespace wrong mode" + namespace.Mode) + } + if mode == "fsdax" { + devicePath = "/dev/" + namespace.BlockDev + } else { + devicePath = "/dev/" + namespace.CharDev + } + return devicePath, namespace.Dev, nil +} + +func (np *NodePmemer) getRegionNamespaceInfo(region string) (*model.PmemRegions, error) { + listCmd := fmt.Sprintf("%s ndctl list -RN -r %s", NsenterCmd, region) + + out, err := Run(listCmd) + if err != nil { + log.Errorf("List NameSpace for region %s error: %v", region, err) + return nil, err + } + regions := &model.PmemRegions{} + err = json.Unmarshal(([]byte)(out), regions) + if len(regions.Regions) == 0 { + log.Errorf("list Namespace for region %s get 0 region, out: %s", region, out) + return nil, errors.New("list Namespace get 0 region by " + region) + } + + if len(regions.Regions[0].Namespaces) != 1 { + log.Errorf("list Namespace for region %s get 0 or multi namespaces", region) + return nil, errors.New("list Namespace for region get 0 or multi namespaces" + region) + } + return regions, nil +} + +// MakeNamespaceMemory ... +func (np *NodePmemer) MakeNamespaceMemory(chardev string) error { + makeCmd := fmt.Sprintf("%s daxctl reconfigure-device -m system-ram %s", NsenterCmd, chardev) + _, err := Run(makeCmd) + return err +} + +// CheckKMEMCreated ... +func (np *NodePmemer) CheckKMEMCreated(chardev string) (bool, error) { + listCmd := fmt.Sprintf("%s daxctl list", NsenterCmd) + out, err := Run(listCmd) + if err != nil { + log.Errorf("CheckKMEMCreated:: List daxctl error: %v", err) + return false, err + } + memList := []*model.DaxctrlMem{} + err = json.Unmarshal(([]byte)(out), &memList) + if err != nil { + return false, err + } + for _, mem := range memList { + if mem.Chardev == chardev && mem.Mode == "system-ram" { + return true, nil + } + } + return false, nil +} diff --git a/pkg/utils/pmem_mock.go b/pkg/utils/pmem_mock.go new file mode 100644 index 0000000..481c15a --- /dev/null +++ b/pkg/utils/pmem_mock.go @@ -0,0 +1,135 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "reflect" + + "github.com/golang/mock/gomock" + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +// MockPmemer .. +type MockPmemer struct { + ctrl *gomock.Controller + recorder *MockPmemerMockRecorder +} + +// MockPmemerMockRecorder ... +type MockPmemerMockRecorder struct { + mock *MockPmemer +} + +// EXPECT ... +func (m *MockPmemer) EXPECT() *MockPmemerMockRecorder { + return m.recorder +} + +// NewMockPmemer ... +func NewMockPmemer(ctrl *gomock.Controller) *MockPmemer { + mock := &MockPmemer{ctrl: ctrl} + mock.recorder = &MockPmemerMockRecorder{mock} + return mock +} + +// GetRegions ... +func (m *MockPmemer) GetRegions() (*model.PmemRegions, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRegions") + ret0, _ := ret[0].(*model.PmemRegions) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRegions ... +func (mr *MockPmemerMockRecorder) GetRegions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRegions", reflect.TypeOf((*MockPmemer)(nil).GetRegions)) +} + +// CreateNamespace ... +func (m *MockPmemer) CreateNamespace(arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateNamespace", arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateNamespace ... +func (mr *MockPmemerMockRecorder) CreateNamespace(arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNamespace", reflect.TypeOf((*MockPmemer)(nil).CreateNamespace), arg1, arg2) +} + +// CheckNamespaceUsed ... +func (m *MockPmemer) CheckNamespaceUsed(arg1 string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckNamespaceUsed", arg1) + ret0, _ := ret[0].(bool) + return ret0 +} + +// CheckNamespaceUsed ... +func (mr *MockPmemerMockRecorder) CheckNamespaceUsed(arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckNamespaceUsed", reflect.TypeOf((*MockPmemer)(nil).CheckNamespaceUsed), arg1) +} + +// GetPmemNamespaceDeivcePath ... +func (m *MockPmemer) GetPmemNamespaceDeivcePath(arg1, arg2 string) (string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPmemNamespaceDeivcePath", arg1, arg2) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetPmemNamespaceDeivcePath ... +func (mr *MockPmemerMockRecorder) GetPmemNamespaceDeivcePath(arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPmemNamespaceDeivcePath", reflect.TypeOf((*MockPmemer)(nil).GetPmemNamespaceDeivcePath), arg1, arg2) +} + +// CheckKMEMCreated ... +func (m *MockPmemer) CheckKMEMCreated(arg1 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckKMEMCreated", arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CheckKMEMCreated ... +func (mr *MockPmemerMockRecorder) CheckKMEMCreated(arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckKMEMCreated", reflect.TypeOf((*MockPmemer)(nil).CheckKMEMCreated), arg1) +} + +// MakeNamespaceMemory ... +func (m *MockPmemer) MakeNamespaceMemory(arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MakeNamespaceMemory", arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// MakeNamespaceMemory ... +func (mr *MockPmemerMockRecorder) MakeNamespaceMemory(arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakeNamespaceMemory", reflect.TypeOf((*MockPmemer)(nil).MakeNamespaceMemory), arg1) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go new file mode 100644 index 0000000..ca59b50 --- /dev/null +++ b/pkg/utils/utils.go @@ -0,0 +1,176 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + "os/exec" + "path/filepath" + "strings" + + "github.com/openyurtio/node-resource-manager/pkg/model" + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // MetadataURL is metadata url + MetadataURL = "http://100.100.100.200/latest/meta-data/" + + // NsenterCmd use to init resource + NsenterCmd = "/usr/bin/nsenter --mount=/proc/1/ns/mnt --ipc=/proc/1/ns/ipc --net=/proc/1/ns/net --uts=/proc/1/ns/uts " +) + +// ErrParse ... +var ErrParse = errors.New("Cannot parse output of blkid") + +//GetMetaData get metadata from ecs meta-server +func GetMetaData(resource string) (string, error) { + resp, err := http.Get(MetadataURL + resource) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} + +// Run run shell command +func Run(cmd string) (string, error) { + out, err := exec.Command("sh", "-c", cmd).CombinedOutput() + if err != nil { + return "", fmt.Errorf("Failed to run cmd: " + cmd + ", with out: " + string(out) + ", with error: " + err.Error()) + } + return string(out), nil +} + +// NodeFilter go through all configmap to find current node config +func NodeFilter(configOperator metav1.LabelSelectorOperator, configKey, configValue string, nodeInfo *v1.Node) bool { + + isMatched := false + switch configOperator { + case metav1.LabelSelectorOpIn: + for key, value := range nodeInfo.Labels { + if key == configKey && value == configValue { + isMatched = true + } + } + case metav1.LabelSelectorOpNotIn: + flag := false + for key, value := range nodeInfo.Labels { + if key == configKey && value == configValue { + flag = true + } + } + if flag == false { + isMatched = true + } + case metav1.LabelSelectorOpExists: + for key := range nodeInfo.Labels { + if key == configKey { + isMatched = true + } + } + case metav1.LabelSelectorOpDoesNotExist: + flag := false + for key := range nodeInfo.Labels { + if key == configKey { + flag = true + } + } + if flag == false { + isMatched = true + } + default: + log.Errorf("Get unsupported operator: %s", configOperator) + } + return isMatched +} + +// ConvertRegion2Namespace ... +func ConvertRegion2Namespace(region string) string { + regionIndex := region[6:] + return fmt.Sprintf("namespace%s.0", regionIndex) +} + +// ConvertNamespace2LVMDevicePath ... +func ConvertNamespace2LVMDevicePath(namespace string, regions *model.PmemRegions) string { + for _, region := range regions.Regions { + for _, actualNamespace := range region.Namespaces { + if actualNamespace.Dev == namespace { + return filepath.Join("/dev", actualNamespace.BlockDev) + } + } + } + return "" +} + +func checkFSType(devicePath string) (string, error) { + // We use `file -bsL` to determine whether any filesystem type is detected. + // If a filesystem is detected (ie., the output is not "data", we use + // `blkid` to determine what the filesystem is. We use `blkid` as `file` + // has inconvenient output. + // We do *not* use `lsblk` as that requires udev to be up-to-date which + // is often not the case when a device is erased using `dd`. + output, err := exec.Command("file", "-bsL", devicePath).CombinedOutput() + if err != nil { + return "", err + } + if strings.TrimSpace(string(output)) == "data" { + return "", nil + } + output, err = exec.Command("blkid", "-c", "/dev/null", "-o", "export", devicePath).CombinedOutput() + if err != nil { + return "", err + } + + lines := strings.Split(string(output), "\n") + for _, line := range lines { + fields := strings.Split(strings.TrimSpace(line), "=") + if len(fields) != 2 { + return "", ErrParse + } + if fields[0] == "TYPE" { + return fields[1], nil + } + } + return "", ErrParse +} + +// IsPart if smallList is part of or equal to largeList, return true; +func IsPart(largeList, smallList []string) bool { + isPartFlag := true + for _, smalltmp := range smallList { + flag := false + for _, largetmp := range largeList { + if smalltmp == largetmp { + flag = true + } + } + if flag == false { + isPartFlag = false + } + } + return isPartFlag +}