diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..ec1f8c2 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,11 @@ +# CHANGELOG + +--- + +### v0.1.0 + +### node-resource-manager + +- Add LVM resource into node resource manager +- Add QuotaPath resource into node resource manager +- Add KMEM resource into node resource manager \ No newline at end of file diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..31941b4 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,5 @@ +# OpenYurt Community Code of Conduct + +Node-resource-manager follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md). + +In cases of abusive, harassing, or any unacceptable behaviors, please don't hesitate to contact the project team at openyurt@gmail.com. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..fd50c12 --- /dev/null +++ b/Makefile @@ -0,0 +1,20 @@ +# Copyright 2021 The OpenYurt Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +.PHONY: build +build: +# options: +# ARCH=amd64 +# VERSION=v1.0 + bash ./build/build.sh $(OPTS) diff --git a/PROJECT b/PROJECT new file mode 100644 index 0000000..25830f5 --- /dev/null +++ b/PROJECT @@ -0,0 +1,3 @@ +version: "1" +domain: openyurt.io +repo: github.com/openyurtio/node-resource-manager diff --git a/README.md b/README.md new file mode 100644 index 0000000..1fcf9e3 --- /dev/null +++ b/README.md @@ -0,0 +1,49 @@ +# openyurtio/node-resource-manager + +English | [简体中文](./README.zh.md) + +Node-resource-manager manages local node resources of OpenYurt cluster in a unified manner. + +It currently manages: +- LVM built on top of block device or pmem device. +- QuotaPath built on top of block device or pmem device. +- Memory built on top of pmem device. + +The majority function consists of: +- Initialize local resources on edge node. +- Update local resources on edge node. + +You can define the spec of local resources by simply modifying the pre-defined ConfigMap. + +Node-resource-manager has the following advantages in terms of compatibility and usability. +- **Easily to use**. Initialization and modification of local resources are easily done by editing the ConfigMap. +- **Easily to integrate**. Node-resource-manager can work together with csi driver, to perform local storage lifecycle management. +- **Platform free**. Node-resource-manager can be running in any kubernetes clusters. + +## Architecture +The component consists of two parts, the first part is the ConfigMap named node-resource-topo in kube-system namespace, +and the second is the node-resource-manager DaemonSet deployed in kube-system namespace. +Node-resource-manager on each node mounts and reads the node-resource-topo ConfigMap to managed local resources. +
+ +
+ +## Getting started + +1. Create node-resource-topo ConfigMap in kube-system namespace. ConfigMap example is in [configmap.md](./docs/configmap.md). +``` +kubectl apply -f deploy/configmap.yaml +``` + +2. Deploy node-resource-manager DaemonSet. +``` +kubectl apply -f deploy/nrm.yaml +``` + +## Developer guide + +Please refer to [developer-guide.md](./docs/developer-guide.md) for developing and building the project. + +## Roadmap + +[2021 Roadmap](docs/roadmap.md) diff --git a/README.zh.md b/README.zh.md new file mode 100644 index 0000000..563ea2a --- /dev/null +++ b/README.zh.md @@ -0,0 +1,53 @@ +# openyurtio/node-resource-manager + +[English](./README.md) | 简体中文 + +node-resource-manager 是用于管理 OpenYurt 集群本地资源的组件，用户可以通过修改集群内 ConfigMap 的定义来动态配置集群内宿主机上的本地资源。 + +管理的本地资源包括： +- 基于块设备或者是持久化内存设备创建的 LVM +- 基于块设备或者是持久化内存设备创建的 QuotaPath + +主要功能包括： +- 初始化节点上本地资源 +- 更新节点上的本地资源 + +主要优点： +- **简单易用**。node-resource-manager 可以仅通过定义 ConfigMap 就完成对集群中的本地资源的初始化和更新 +- **易于集成**。 node-resource-manager 可以与 csi 插件集成来完成 kubernetes 集群中的相关本地资源的生命周期管理 +- **与云平台无关**。 node-resource-manager 可以轻松部署在任何公共云 Kubernetes 服务中。 + +## 架构 + +该组件主要包含两个部分， 一个是定义在集群中 kube-system namespace 的 node-resource-topo ConfigMap, +一个是部署在集群中 kube-system namespace 下面的 node-resource-manager Daemonset, +每个 Node 节点上的 node-resource-manager 通过挂载 node-resource-topo ConfigMap 的方式生产并管理用户定义的本地资源。 +
+ +
+ +## 开始使用 + +1. 在 Kubernetes kube-system namespace 下定义 node-resource-topo ConfigMap, 该 ConfigMap 用于定义集群中需要自动生成并管理的节点本地资源. 关于如何创建一个ConfigMap，请参见 ConfigMap [定义](./docs/configmap.zh.md) + +``` +kubectl apply -f deploy/configmap.yaml +``` + +2. 在 Kubernetes 集群中创建 node-resource-manager Daemonset。 +``` +kubectl apply -f deploy/nrm.yaml +``` + +3. 检查在 ConfigMap 上定义的资源是否都已经在对应的节点上被正确的创建。 + + +4. 配合 [alibaba-local-csi-plugin](https://help.aliyun.com/document_detail/178472.html?spm=a2c4g.11186623.6.844.13a019caYIiivY) 插件在集群中动态创建本地资源 pvc/pv, 供 pod 进行挂载使用。 + +## 开发指南 + +请参考 [developer-guide.md](./docs/developer-guide.md) 进行本项目的开发和构建。 + +## 发展规划 + +[2021年 发展规划](docs/roadmap.md) \ No newline at end of file diff --git a/build/build.sh b/build/build.sh new file mode 100755 index 0000000..cd403b1 --- /dev/null +++ b/build/build.sh @@ -0,0 +1,118 @@ +# Copyright 2021 The OpenYurt Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env bash +set -x + +# usage: +# ./build.sh ARCH=amd64 VERSION=v1.0 + +NRM_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" +NRM_BUILD_DIR=${NRM_ROOT}/build +NRM_OUTPUT_DIR=${NRM_BUILD_DIR}/_output +NRM_BUILD_IMAGE="golang:1.13.3-alpine" + +GIT_VERSION="v1.0" +GIT_VERSION=(${VERSION:-${GIT_VERSION}}) +GIT_SHA=`git rev-parse --short HEAD || echo "HEAD"` +GIT_BRANCH=`git rev-parse --abbrev-ref HEAD 2>/dev/null` +BUILD_TIME=`date "+%Y-%m-%d-%H:%M:%S"` + +IMG_REPO="openyurt/node-resource-manager" +IMG_VERSION=${GIT_VERSION}-${GIT_SHA} + +readonly -a SUPPORTED_ARCH=( + amd64 + arm + arm64 +) + +readonly -a target_arch=(${ARCH:-${SUPPORTED_ARCH[@]}}) + +function build_multi_arch_binaries() { + local docker_run_opts=( + "-i" + "--rm" + "--network host" + "-v ${NRM_ROOT}:/opt/src" + "--env CGO_ENABLED=0" + "--env GOOS=linux" + "--env GIT_VERSION=${GIT_VERSION}" + "--env GIT_SHA=${GIT_SHA}" + "--env GIT_BRANCH=${GIT_BRANCH}" + "--env BUILD_TIME=${BUILD_TIME}" + ) + # use goproxy if build from inside mainland China + [[ $region == "cn" ]] && docker_run_opts+=("--env GOPROXY=https://goproxy.cn") + + local docker_run_cmd=( + "/bin/sh" + "-xe" + "-c" + ) + + local sub_commands="sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories; \ + apk --no-cache add bash; \ + cd /opt/src; " + for arch in ${target_arch[@]}; do + sub_commands+="CGO_ENABLED=0 GOOS=linux GOARCH='${arch}' go build \ + -ldflags '-X main._BRANCH_=${GIT_BRANCH} -X main._VERSION_=${IMG_VERSION} -X main._BUILDTIME_=${BUILD_TIME}' -o build/_output/nrm.${arch} main.go; " + done + + docker run ${docker_run_opts[@]} ${NRM_BUILD_IMAGE} ${docker_run_cmd[@]} "${sub_commands}" +} + +function build_images() { + for arch in ${target_arch[@]}; do + local docker_file_path=${NRM_BUILD_DIR}/Dockerfile.$arch + local docker_build_path=${NRM_BUILD_DIR} + local nrm_image=$IMG_REPO:${IMG_VERSION}.${arch} + local base_image + case $arch in + amd64) + base_image="amd64/alpine:3.10" + ;; + arm64) + base_image="arm64v8/alpine:3.10" + ;; + arm) + base_image="arm32v7/alpine:3.10" + ;; + *) + echo unknown arch $arch + exit 1 + esac + cat << EOF > $docker_file_path +FROM ${base_image} +LABEL maintainers="OpenYurt Authors" +LABEL description="OpenYurt Node Resource Manager" + +RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories +RUN apk update && apk add --no-cache ca-certificates file util-linux lvm2 xfsprogs e2fsprogs blkid +COPY entrypoint.sh /entrypoint.sh +COPY _output/nrm.${arch} /bin/nrm +RUN chmod +x /bin/nrm && chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] +EOF + docker build --no-cache -t $nrm_image -f $docker_file_path $docker_build_path + docker save $nrm_image > ${NRM_OUTPUT_DIR}/node-resource-manager-${arch}.tar + done +} + +rm -rf ${NRM_OUTPUT_DIR} +mkdir -p ${NRM_OUTPUT_DIR} +umask 0022 +build_multi_arch_binaries +build_images diff --git a/build/entrypoint.sh b/build/entrypoint.sh new file mode 100644 index 0000000..1ae02eb --- /dev/null +++ b/build/entrypoint.sh @@ -0,0 +1,30 @@ +#!/bin/sh + +if uname -a | grep -i -q ubuntu; then + lvmLine=`/usr/bin/nsenter --mount=/proc/1/ns/mnt dpkg --get-selections lvm2 | grep install -w -i | wc -l` + if [ "$lvmLine" = "0" ]; then + /usr/bin/nsenter --mount=/proc/1/ns/mnt apt install lvm2 -y + fi +else + lvmLine=`/usr/bin/nsenter --mount=/proc/1/ns/mnt rpm -qa lvm2 | wc -l` + if [ "$lvmLine" = "0" ]; then + /usr/bin/nsenter --mount=/proc/1/ns/mnt yum install lvm2 -y + fi +fi + +if [ "$lvmLine" = "0" ]; then + /usr/bin/nsenter --mount=/proc/1/ns/mnt sed -i 's/udev_sync\ =\ 0/udev_sync\ =\ 1/g' /etc/lvm/lvm.conf + /usr/bin/nsenter --mount=/proc/1/ns/mnt sed -i 's/udev_rules\ =\ 0/udev_rules\ =\ 1/g' /etc/lvm/lvm.conf + /usr/bin/nsenter --mount=/proc/1/ns/mnt systemctl restart lvm2-lvmetad.service + echo "install lvm and starting..." +else + udevLine=`/usr/bin/nsenter --mount=/proc/1/ns/mnt cat /etc/lvm/lvm.conf | grep "udev_sync = 0" | wc -l` + if [ "$udevLine" != "0" ]; then + /usr/bin/nsenter --mount=/proc/1/ns/mnt sed -i 's/udev_sync\ =\ 0/udev_sync\ =\ 1/g' /etc/lvm/lvm.conf + /usr/bin/nsenter --mount=/proc/1/ns/mnt sed -i 's/udev_rules\ =\ 0/udev_rules\ =\ 1/g' /etc/lvm/lvm.conf + /usr/bin/nsenter --mount=/proc/1/ns/mnt systemctl restart lvm2-lvmetad.service + echo "update lvm.conf file: udev_sync from 0 to 1, udev_rules from 0 to 1" + fi +fi + +/bin/nrm $@ diff --git a/deploy/configmap.yaml b/deploy/configmap.yaml new file mode 100644 index 0000000..bbd66c0 --- /dev/null +++ b/deploy/configmap.yaml @@ -0,0 +1,61 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + volumegroup: |- + volumegroup: + - name: volumegroup1 + key: kubernetes.io/hostname + operator: In + value: cn-zhangjiakou.192.168.3.114 + topology: + type: device + devices: + - /dev/vdb + - /dev/vdc + + - name: volumegroup1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.35 + topology: + type: pmem + regions: + - region0 + + quotapath: |- + quotapath: + - name: /mnt/path1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.35 + topology: + type: device + options: prjquota + fstype: ext4 + devices: + - /dev/vdb + + - name: /mnt/path2 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.36 + topology: + type: pmem + options: prjquota,shared + fstype: ext4 + regions: + - region0 + + memory: |- + memory: + - name: test1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.37 + topology: + type: pmem + regions: + - region0 diff --git a/deploy/nrm.yaml b/deploy/nrm.yaml new file mode 100644 index 0000000..ea72110 --- /dev/null +++ b/deploy/nrm.yaml @@ -0,0 +1,92 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: node-resource-manager + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node-resource-manager +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: node-resource-manager-binding +subjects: + - kind: ServiceAccount + name: node-resource-manager + namespace: kube-system +roleRef: + kind: ClusterRole + name: node-resource-manager + apiGroup: rbac.authorization.k8s.io +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: node-resource-manager + namespace: kube-system +spec: + selector: + matchLabels: + app: node-resource-manager + template: + metadata: + labels: + app: node-resource-manager + spec: + tolerations: + - operator: "Exists" + priorityClassName: system-node-critical + serviceAccountName: node-resource-manager + hostNetwork: true + hostPID: true + containers: + - name: node-resource-manager + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: openyurt/node-resource-manager:v1.0 + imagePullPolicy: "Always" + args: + - "--nodeid=$(KUBE_NODE_NAME)" + env: + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + volumeMounts: + - mountPath: /dev + mountPropagation: "HostToContainer" + name: host-dev + - mountPath: /var/log/ + name: host-log + - name: etc + mountPath: /host/etc + - name: config + mountPath: /etc/unified-config + volumes: + - name: host-dev + hostPath: + path: /dev + - name: host-log + hostPath: + path: /var/log/ + - name: etc + hostPath: + path: /etc + - name: config + configMap: + name: node-resource-topo diff --git a/docs/configmap.md b/docs/configmap.md new file mode 100644 index 0000000..d69df16 --- /dev/null +++ b/docs/configmap.md @@ -0,0 +1,148 @@ +# ConfigMap Intro + +"node-resource-topo" ConfigMap defines the resource spec in the whole cluster. +Node-resource-manager on each node will perform create or update action according the resource definition in ConfigMap. + +## Supported Node Resources + +### LVM +- LVM VolumeGroup creation, according to the VG defined in ConfigMap; +- Create PV from local disk and add into VG; +- Don't support deletion or shrink of VG, to avoid data lost risk; + +### QuotaPath +- QuotaPath creation and mount, according to the definition in ConfigMap; +- Don't support deletion or shrink of QuotaPath, to avoid data lost risk; + +### PMEM +- Format pmem device as local memory, it can be later used by pod; + +## How to define target node +We use the kubernetes label selector to choose target node: +```yaml +key: kubernetes.io/hostname +operator: In +value: xxxxx +``` +- key: match the key in Node labels; +- operator: Labels selector operator, + - In: matched only if current value equals to the value of the key in Node Labels; + - NotIn: matched only if current value not equals to the value of key in the Node Labels; + - Exists: matched if current key exists in Node Labels; + - DoesNotExist: matched if current key not exists in Node Labels; +- value: match the corresponding value of the key in Node labels; + +## Example + +### LVM +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + volumegroup: |- + volumegroup: + - name: volumegroup1 + key: kubernetes.io/hostname + operator: In + value: cn-zhangjiakou.192.168.3.114 + topology: + type: device + devices: + - /dev/vdb + - /dev/vdc + + - name: volumegroup2 + key: kubernetes.io/nodetype + operator: NotIn + value: localdisk + topology: + type: alibabacloud-local-disk + + - name: volumegroup1 + key: kubernetes.io/hostname + operator: Exists + value: cn-beijing.192.168.3.35 + topology: + type: pmem + regions: + - region0 +``` +The above ConfigMap defines: +- select nodes with label `kubernetes.io/hostname: cn-zhangjiakou.192.168.3.114`, and create LVM VolumeGroup named `volumegroup1` from device `/dev/vdb` and `/dev/vdc`; +- select nodes without label `kubernetes.io/nodetype: localdisk`, and create LVM VolumeGroup named `volumegroup2` from all ecs cloud disks; +- select nodes with label `kubernetes.io/hostname: cn-beijing.192.168.3.35`, and create LVM VolumeGroup named `volumegroup1` from pmem in `region0`; + +LVM currently supports three types of devices: +- `type: device` define lvm on top of local block devices, the volumegroup's name is specified in `name` field; +- `type: alibabacloud-local-disk` define lvm on top of attached cloud disks for alicloud ecs, the volumegroup's name is specified in `name` field; +- `type: pmem` define lvm on top of local pmem resources, the volumegroup's name is specified in `name` field, the pmem regions is specified in `regions` field; + +### QuotaPath +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + quotapath: |- + quotapath: + - name: /mnt/path1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.35 + topology: + type: device + options: prjquota + fstype: ext4 + devices: + - /dev/vdb + + - name: /mnt/path2 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.36 + topology: + type: pmem + options: prjquota,shared + fstype: ext4 + regions: + - region0 +``` +The above ConfigMap defines: +- select nodes with label `kubernetes.io/hostname: cn-beijing.192.168.3.35`, and create quota path `/mnt/path1` mounted from device `/dev/vdb`, and format to ext4 filesystem with prjquota option; +- select nodes with label `kubernetes.io/hostname: cn-beijing.192.168.3.36`, and create quota path `/mnt/path2` mounted from pmem in `region0`, and format to ext4 filesystem with prjquota, shared option; + +QuotaPath currently supports two types of devices: +- `type: device` define quota path on top of local block device, the quota path is specified in `name` field: + - options: mount options, `prjquota` is mandatory; + - fstype: filesystem type, ext4 is used by default; + - devices: block device to be mounted, you should specify only one device; +- `type: pmem` define quota path on top of local pmem resources, the quota path is specified in `name` field, you can speficy pmem regions in `regions` field; + +### PMEM +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + memory: |- + memory: + - name: test1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.37 + topology: + type: pmem + regions: + - region0 +``` +The above ConfigMap defines: +- select nodes with label `kubernetes.io/hostname: cn-beijing.192.168.3.37`, and create memory from pmem in `region0`; + +PMEM only support `type: pmem`, you can speficy pmem regions in `regions` field, name field is only a symbol and has no actual usage. diff --git a/docs/configmap.zh.md b/docs/configmap.zh.md new file mode 100644 index 0000000..444f4e8 --- /dev/null +++ b/docs/configmap.zh.md @@ -0,0 +1,152 @@ +# ConfigMap 介绍 + +"node-resource-topo" ConfigMap 定义整个集群的设备拓扑，每个节点的 node-resource-manager 会根据 ConfigMap 中的资源定义决定是否创建或者更新资源； + +## 节点资源支持 + +### LVM +- 根据 ConfigMap 中定义，创建 LVM VolumeGroup； +- 在 VG 中添加、扩展 PV；根据 ConfigMap 中定义的 VG 信息，判断是否执行 VG 扩容命令； +- 考虑数据安全性，不支持 VG 的删除、缩容操作； + +### QuotaPath +- QuotaPath 的创建, 根据 ConfigMap 中的定义来初始化相关本地资源设备以 QuotaPath 的形式挂载到指定路径上； +- 不支持相关 QuotaPath 的变更, 删除等操作； + +### PMEM +- 支持将持久化内存设备初始化为内存格式。后续可以直接被挂载到pod内部目录中使用； + +## 如何定义节点 + +我们通过如下三个 key/value 来共同定义资源所在的节点： + +```yaml +key: kubernetes.io/hostname +operator: In +value: xxxxx +``` +- key: 匹配 Kubernetes Node Labels 中的 key 的值； +- operator: Kubernetes 定义的 Labels selector operator，主要包含如下四种操作符； + - In: 只有 value 的值与 Node 上 Labels key 对应的值相同的时候才会匹配； + - NotIn: 只有 value 的值与 Node 上 Labels key 对应的 value 的值 ***不*** 相同的时候才会匹配； + - Exists: 只要 Node 的 Labels 上存在 Key 就会匹配； + - DoesNotExist: 只要 Node 的 Labels 上 ***不*** 存在 Key 就会匹配； +- value: 匹配 Kubernetes Node Labels 的 key 对应的 value 的值； + +## 常用模板用例及说明 + +### LVM + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + volumegroup: |- + volumegroup: + - name: volumegroup1 + key: kubernetes.io/hostname + operator: In + value: cn-zhangjiakou.192.168.3.114 + topology: + type: device + devices: + - /dev/vdb + - /dev/vdc + + - name: volumegroup2 + key: kubernetes.io/nodetype + operator: NotIn + value: localdisk + topology: + type: alibabacloud-local-disk + + - name: volumegroup1 + key: kubernetes.io/hostname + operator: Exists + value: cn-beijing.192.168.3.35 + topology: + type: pmem + regions: + - region0 +``` +上面的 ConfigMap 定义了 +1. 在拥有 Label key 等于 kubernetes.io/hostname 并且 Label key value 等于 cn-zhangjiakou.192.168.3.114 的 Node 上创建一个名字为 volumegroup1 的 LVM VolumeGroup, 这个 VolumeGroup 由宿主机上的 ```/dev/vdb, /dev/vdc``` 两个块设备组成 +2. 在拥有 Label key 等于 kubernetes.io/hostname 并且 Label key value 不等于 localdisk 的 Node 上创建一个名字为 volumegroup2 的 LVM VolumeGroup，这个 LVM VolumeGroup 由宿主机上的所有本地盘组成 +3. 在拥有 Label key 等于 kubernetes.io/hostname 的 Node 上创建一个名字为 volumegroup1 的 LVM VolumeGroup, 这个 VolumeGroup 由宿主机上的 Pmem 设备 region0 组成 + +lvm目前仅支持三种定义资源拓扑的方式 +- 当定义 ```type: device``` 的时候是通过 nrm 所在宿主机上存在的块设备 devices 进行 lvm 的声明，声明的块设备会组成一个 volumegroup, volumegroup 的名字由 name 字段指定，供后续应用启动时分配 logical volume。```type: device``` 类型与下面的 ```devices``` 字段绑定； +- 当定义 ```type: alibabacloud-local-disk``` 的时候指的是使用 urm 所在宿主机上所有的本地盘 (选择 ecs 类型带有本地盘的 instance , 例如 本地 SSD 型 i2, 手动挂载到 ecs 上的云盘不是本地盘) 共同创建一个 名称为 name 值的 volumegroup； +- 当定义 ```type: pmem``` 的时候是使用 urm 所在宿主机上的 pmem 资源创建一个名称为 name 值的 volumegroup, 其中 regions 可以指定当前机器上多个 pmem region 资源。```type: pmem``` 类型与下面的 ```regions``` 字段绑定。 + +### QuotaPath + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + quotapath: |- + quotapath: + - name: /mnt/path1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.35 + topology: + type: device + options: prjquota + fstype: ext4 + devices: + - /dev/vdb + + - name: /mnt/path2 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.36 + topology: + type: pmem + options: prjquota,shared + fstype: ext4 + regions: + - region0 +``` +上面的 ConfigMap 定义了 +1. 在拥有 Label key 等于 kubernetes.io/hostname 并且 Label key value 等于 cn-beijing.192.168.3.35 的 Node 上的 ```/mnt/path1``` 上以 prjquota 类型挂载 ```/dev/vdb``` 的块设备， 并且格式化成 project quota ext4 格式。 +2. 在拥有 Label key 等于 kubernetes.io/hostname 并且 Label key value 等于 cn-beijing.192.168.3.36 的 Node 上的 ```/mnt/path2``` 上以prjquota 类型挂载 ```region0``` 的 Pmem 设备， 并且格式化成 project quota ext4 格式。 + +QuotaPath 类型的本地资源仅支持两种定义资源拓扑的方式： +- 当定义 ```type: device``` 的时候，使用的是 nrm 所在宿主机的块设备进行 QuotaPath 的初始化，初始化路径是 name 字段定义的值, 下面解释下其他几个字段的定义： + - options: 块设备在被挂载的时候使用的参数。无特殊需求使用例子中提供的参数即可； + - fstype: 格式化块设备使用的文件系统，默认使用 ext4； + - devices：挂载使用的块设备，只可以声明一个； + +### pmem + +```yaml +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-resource-topo + namespace: kube-system +data: + memory: |- + memory: + - name: test1 + key: kubernetes.io/hostname + operator: In + value: cn-beijing.192.168.3.37 + topology: + type: pmem + regions: + - region0 +``` + +上面的 ConfigMap 定义了 +1. 在拥有 Label key 等于 kubernetes.io/hostname 并且 Label key value 等于 cn-beijing.192.168.3.37 的 Node 上用 Pmem 的 Region0 设备初始化一个内存格式的 Pmem 设备。 + +持久化内存类型本地资源只支持 pmem type 的挂载，其中 regions 代表需要被格式化成 memory 的 pmem 设备，name 字段在这里只起到标识作用，无实际意义。 diff --git a/docs/developer-guide.md b/docs/developer-guide.md new file mode 100644 index 0000000..602ae96 --- /dev/null +++ b/docs/developer-guide.md @@ -0,0 +1,15 @@ +# Developer Guide + +There's a `Makefile` in the root folder. + + + +- Build docker images for specific architecture, supported architectures are `amd64`,`arm`,`arm64`: +```bash +ARCH=amd64 make build +``` + +- Build all docker images for all supported architectures. +```bash +make build +``` diff --git a/docs/images/node-resource-manager.png b/docs/images/node-resource-manager.png new file mode 100644 index 0000000..d8c5a1e Binary files /dev/null and b/docs/images/node-resource-manager.png differ diff --git a/docs/roadmap.md b/docs/roadmap.md new file mode 100644 index 0000000..bf845cb --- /dev/null +++ b/docs/roadmap.md @@ -0,0 +1,3 @@ +# Roadmap + +This document outlines the development roadmap for the node-resource-manager project. diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..270ae8e --- /dev/null +++ b/go.mod @@ -0,0 +1,23 @@ +module github.com/openyurtio/node-resource-manager + +go 1.15 + +require ( + github.com/aliyun/alibaba-cloud-sdk-go v1.61.895 + github.com/golang/mock v1.3.1 + github.com/imdario/mergo v0.3.11 // indirect + github.com/sirupsen/logrus v1.7.0 + github.com/stretchr/testify v1.6.1 + golang.org/x/net v0.0.0-20210119194325-5f4716e94777 // indirect + golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 // indirect + gopkg.in/yaml.v2 v2.4.0 + k8s.io/api v0.20.2 + k8s.io/apimachinery v0.20.2 + k8s.io/client-go v11.0.0+incompatible + k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 +) + +replace ( + k8s.io/api => k8s.io/api v0.19.2 + k8s.io/client-go => k8s.io/client-go v0.19.2 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..74780d2 --- /dev/null +++ b/go.sum @@ -0,0 +1,388 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.51.0/go.mod h1:hWtGJ6gnXH+KgDv+V0zFGDvpi07n3z8ZNj3T1RW0Gcw= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/go-autorest/autorest v0.9.0/go.mod h1:xyHB1BMZT0cuDHU7I0+g046+BFDTQ8rEZB0s4Yfa6bI= +github.com/Azure/go-autorest/autorest v0.9.6/go.mod h1:/FALq9T/kS7b5J5qsQ+RSTUdAmGFqi0vUdVNNx8q630= +github.com/Azure/go-autorest/autorest/adal v0.5.0/go.mod h1:8Z9fGy2MpX0PvDjB1pEgQTmVqjGhiHBW7RJJEciWzS0= +github.com/Azure/go-autorest/autorest/adal v0.8.2/go.mod h1:ZjhuQClTqx435SRJ2iMlOxPYt3d2C/T/7TiQCVZSn3Q= +github.com/Azure/go-autorest/autorest/date v0.1.0/go.mod h1:plvfp3oPSKwf2DNjlBjWF/7vwR+cUD/ELuzDCXwHUVA= +github.com/Azure/go-autorest/autorest/date v0.2.0/go.mod h1:vcORJHLJEh643/Ioh9+vPmf1Ij9AEBM5FuBIXLmIy0g= +github.com/Azure/go-autorest/autorest/mocks v0.1.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.2.0/go.mod h1:OTyCOPRA2IgIlWxVYxBee2F5Gr4kF2zd2J5cFRaIDN0= +github.com/Azure/go-autorest/autorest/mocks v0.3.0/go.mod h1:a8FDP3DYzQ4RYfVAxAN3SVSiiO77gL2j2ronKKP0syM= +github.com/Azure/go-autorest/logger v0.1.0/go.mod h1:oExouG+K6PryycPJfVSxi/koC6LSNgds39diKLz7Vrc= +github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbtp2fGCgRFtBroKn4Dk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.895 h1:8OvEa8rSbkUzjBNTj3vV9mJx+i8Tw6YS1J26FyBURDw= +github.com/aliyun/alibaba-cloud-sdk-go v1.61.895/go.mod h1:pUKYbK5JQ+1Dfxk80P0qxGqe5dkxDoabbZS7zOcouyA= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0 h1:QvGt2nLcHH0WK9orKa+ppBPAxREcH364nPUedEpK0TY= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gnostic v0.4.1 h1:DLJCy1n/vrD4HPjOvYcT8aYQXpPIzoRZONaYwyycI+I= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.11 h1:3tnifQM4i+fbajXKBHXWEH+KvNHqojZ778UH75j3bGA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6 h1:pE8b58s1HRDMi8RDc79m0HISf9D4TzseP40cEA6IGfs= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 h1:nxC68pudNYkKU6jWhgrqdreuFiOQWj1Fs7T3VrH4Pjw= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= +golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/ini.v1 v1.42.0 h1:7N3gPTt50s8GuLortA00n8AqRTk75qOP98+mTPpgzRk= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.19.2 h1:q+/krnHWKsL7OBZg/rxnycsl9569Pud76UJ77MvKXms= +k8s.io/api v0.19.2/go.mod h1:IQpK0zFQ1xc5iNIQPqzgoOwuFugaYHK4iCknlAQP9nI= +k8s.io/apimachinery v0.19.2/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA= +k8s.io/apimachinery v0.20.2 h1:hFx6Sbt1oG0n6DZ+g4bFt5f6BoMkOjKWsQFu077M3Vg= +k8s.io/apimachinery v0.20.2/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= +k8s.io/client-go v0.19.2 h1:gMJuU3xJZs86L1oQ99R4EViAADUPMHHtS9jFshasHSc= +k8s.io/client-go v0.19.2/go.mod h1:S5wPhCqyDNAlzM9CnEdgTGV4OqhsW3jGO1UM1epwfJA= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0 h1:7+X0fUguPyrKEC4WjH8iGDg3laWgMo5tMnRTIGTTxGQ= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/utils v0.0.0-20200729134348-d5654de09c73/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009 h1:0T5IaWHO3sJTEmCP6mUlBvMukxPKUQWqiI/YuiBNMiQ= +k8s.io/utils v0.0.0-20210111153108-fddb29f9d009/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2 h1:YHQV7Dajm86OuqnIR6zAelnDWBRjo+YhYV9PmGrh1s8= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= diff --git a/main.go b/main.go new file mode 100644 index 0000000..052179a --- /dev/null +++ b/main.go @@ -0,0 +1,123 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "flag" + "io" + "os" + "strings" + "time" + + "github.com/openyurtio/node-resource-manager/pkg/manager" + "github.com/openyurtio/node-resource-manager/pkg/signals" + log "github.com/sirupsen/logrus" +) + +const ( + // LogfilePrefix prefix of log file + LogfilePrefix = "/var/log/openyurt/" + // MBSIZE MB size + MBSIZE = 1024 * 1024 +) + +// BRANCH is NRM Branch +var _BRANCH_ = "" + +// VERSION is NRM Version +var _VERSION_ = "" + +// BUILDTIME is NRM Buildtime +var _BUILDTIME_ = "" + +var ( + nodeID = flag.String("nodeid", "", "node id") + cmName = flag.String("cm-name", "node-resource-manager", "used configmap name") + cmNameSpace = flag.String("cm-namespace", "kube-system", "used configmap namespace") + logLevel = flag.String("log-level", "Info", "Set Log Level") + updateInterval = flag.Int("update-interval", 30, "Node Storage update internal time(s)") + masterURL = flag.String("master", "", "The address of the Kubernetes API server (https://hostname:port, overrides any value in kubeconfig)") + kubeconfig = flag.String("kubeconfig", "", "Path to kubeconfig file with authorization and master location information") +) + +// main +func main() { + flag.Parse() + + // set log config + setLogAttribute("unified-resource-manager") + log.Infof("Unified Resource Manager, Version: %s, Build Time: %s", _VERSION_, _BUILDTIME_) + + // new signal handler + stopCh := signals.SetupSignalHandler() + + // New Controller Manager + manager := manager.NewManager(*nodeID, *cmName, *cmNameSpace, *updateInterval, *masterURL, *kubeconfig) + manager.Run(stopCh) + + os.Exit(0) +} + +func init() { + flag.Set("logtostderr", "true") +} + +// rotate log file by 2M bytes +// default print log to stdout and file both. +func setLogAttribute(driver string) { + logType := os.Getenv("LOG_TYPE") + logType = strings.ToLower(logType) + if logType != "stdout" && logType != "host" { + logType = "both" + } + if logType == "stdout" { + return + } + + os.MkdirAll(LogfilePrefix, os.FileMode(0755)) + logFile := LogfilePrefix + driver + ".log" + f, err := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + os.Exit(1) + } + + // rotate the log file if too large + if fi, err := f.Stat(); err == nil && fi.Size() > 2*MBSIZE { + f.Close() + timeStr := time.Now().Format("-2006-01-02-15:04:05") + timedLogfile := LogfilePrefix + driver + timeStr + ".log" + os.Rename(logFile, timedLogfile) + f, err = os.OpenFile(logFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666) + if err != nil { + os.Exit(1) + } + } + if logType == "both" { + mw := io.MultiWriter(os.Stdout, f) + log.SetOutput(mw) + } else { + log.SetOutput(f) + } + + logLevelLow := strings.ToLower(*logLevel) + if logLevelLow == "debug" { + log.SetLevel(log.DebugLevel) + } else if logLevelLow == "warning" { + log.SetLevel(log.WarnLevel) + } + log.Infof("Set Log level to %s...", logLevelLow) +} diff --git a/pkg/config/cloud.go b/pkg/config/cloud.go new file mode 100644 index 0000000..54df3d3 --- /dev/null +++ b/pkg/config/cloud.go @@ -0,0 +1,104 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "encoding/json" + "os" + "path/filepath" + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/openyurtio/node-resource-manager/pkg/utils" + log "github.com/sirupsen/logrus" +) + +func newEcsClient(accessKeyID, accessKeySecret, accessToken string) (ecsClient *ecs.Client) { + var err error + regionID, _ := utils.GetMetaData(RegionIDTag) + if accessToken == "" { + ecsClient, err = ecs.NewClientWithAccessKey(regionID, accessKeyID, accessKeySecret) + if err != nil { + return nil + } + } else { + ecsClient, err = ecs.NewClientWithStsToken(regionID, accessKeyID, accessKeySecret, accessToken) + if err != nil { + return nil + } + } + return +} + +// GetDefaultAK read default ak from local file or from STS +func GetDefaultAK() (string, string, string) { + accessKeyID, accessSecret := GetLocalAK() + + accessToken := "" + if accessKeyID == "" || accessSecret == "" { + accessKeyID, accessSecret, accessToken = GetSTSAK() + } + + return accessKeyID, accessSecret, accessToken +} + +func GetLocalAK() (string, string) { + var accessKeyID, accessSecret string + // first check if the environment setting + accessKeyID = os.Getenv("ACCESS_KEY_ID") + accessSecret = os.Getenv("ACCESS_KEY_SECRET") + if accessKeyID != "" && accessSecret != "" { + return accessKeyID, accessSecret + } + + return accessKeyID, accessSecret +} + +// RoleAuth define STS Token Response +type RoleAuth struct { + AccessKeyID string + AccessKeySecret string + Expiration time.Time + SecurityToken string + LastUpdated time.Time + Code string +} + +// GetSTSAK get STS AK and token from ecs meta server +func GetSTSAK() (string, string, string) { + roleAuth := RoleAuth{} + subpath := "ram/security-credentials/" + roleName, err := utils.GetMetaData(subpath) + if err != nil { + log.Errorf("GetSTSToken: request roleName with error: %s", err.Error()) + return "", "", "" + } + + fullPath := filepath.Join(subpath, roleName) + roleInfo, err := utils.GetMetaData(fullPath) + if err != nil { + log.Errorf("GetSTSToken: request roleInfo with error: %s", err.Error()) + return "", "", "" + } + + err = json.Unmarshal([]byte(roleInfo), &roleAuth) + if err != nil { + log.Errorf("GetSTSToken: unmarshal roleInfo: %s, with error: %s", roleInfo, err.Error()) + return "", "", "" + } + return roleAuth.AccessKeyID, roleAuth.AccessKeySecret, roleAuth.SecurityToken +} diff --git a/pkg/config/global.go b/pkg/config/global.go new file mode 100644 index 0000000..a9a76ca --- /dev/null +++ b/pkg/config/global.go @@ -0,0 +1,71 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "context" + + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" +) + +// GlobalConfig save global values for plugin +type GlobalConfig struct { + //EcsClient *ecs.Client + //Region string + NodeInfo *v1.Node + KubeClient *kubernetes.Clientset +} + +var ( + GlobalConfigVar GlobalConfig +) + +const ( + // RegionIDTag is region id + RegionIDTag = "region-id" + // InstanceIDTag is instance id + InstanceIDTag = "instance-id" +) + +// GlobalConfigSet set Global Config +func GlobalConfigSet(nodeID, masterURL, kubeconfig string) { + + // Global Configs Set + cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig) + if err != nil { + log.Fatalf("Error building kubeconfig: %s", err.Error()) + } + kubeClient, err := kubernetes.NewForConfig(cfg) + if err != nil { + log.Fatalf("Error building kubernetes clientset: %s", err.Error()) + } + + node, err := kubeClient.CoreV1().Nodes().Get(context.Background(), nodeID, metav1.GetOptions{}) + if err != nil { + log.Fatal("Error get current node info: %s", err.Error()) + } + + // Global Config Set + GlobalConfigVar = GlobalConfig{ + KubeClient: kubeClient, + NodeInfo: node, + } +} diff --git a/pkg/manager/manager.go b/pkg/manager/manager.go new file mode 100644 index 0000000..df583ec --- /dev/null +++ b/pkg/manager/manager.go @@ -0,0 +1,113 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package manager + +import ( + "context" + "time" + + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/manager/memory" + "github.com/openyurtio/node-resource-manager/pkg/manager/quotapath" + "github.com/openyurtio/node-resource-manager/pkg/manager/volumegroup" + log "github.com/sirupsen/logrus" + "k8s.io/client-go/kubernetes" +) + +// Manager interface define local resource manager's action +type Manager interface { + AnalyseConfigMap() error + ApplyResourceDiff() error +} + +// UnifiedResourceManager is global resource manager struct +type UnifiedResourceManager struct { + KubeClientSet *kubernetes.Clientset + UpdateInterval int + NodeID string +} + +// NewDriver create a cpfs driver object +func NewManager(nodeID, cmName, cmNameSpace string, updateInterval int, masterURL, kubeconfig string) *UnifiedResourceManager { + manager := &UnifiedResourceManager{ + NodeID: nodeID, + UpdateInterval: updateInterval, + } + // Config GlobalVar + config.GlobalConfigSet(nodeID, masterURL, kubeconfig) + + return manager +} + +// Run Start the UnifiedResourceManager +// First will check UnifiedResource exist or not; +// Update UnifiedResource every internal seconds, the total/free capacity, storage status; +// Maintain Unified Resource, Like: local disk in alibaba cloud cluster. +func (urm *UnifiedResourceManager) Run(stopCh <-chan struct{}) { + ctx := context.Background() + + // Create UnifiedResource CR if not exist + urm.CreateUnifiedResourceCRD(ctx) + + // Maintain VolumeGroup if set in configMap + go urm.BuildUnifiedResource() + + // UpdateUnifiedStorage + // go wait.Until(urm.RecordUnifiedResources, time.Duration(urm.UpdateInterval)*time.Second, stopCh) + + log.Infof("Starting to update node storage on %s...", urm.NodeID) + <-stopCh + log.Infof("Stop to update node storage...") +} + +// CreateUnifiedResourceCRD ... +func (urm *UnifiedResourceManager) CreateUnifiedResourceCRD(ctx context.Context) error { + return nil +} + +// BuildUnifiedResource ... +func (urm *UnifiedResourceManager) BuildUnifiedResource() { + log.Infof("BuildUnifiedResource:: Starting to maintain unified storage...") + rms := []Manager{volumegroup.NewResourceManager(), quotapath.NewResourceManager(), memory.NewResourceManager()} + + for { + for _, rm := range rms { + err := BuildResource(rm) + if err != nil { + continue + } + } + time.Sleep(time.Duration(20) * time.Second) + } +} + +// BuildResource ... +func BuildResource(m Manager) error { + + // Get Desired VolumeGroup from ConfigMap + err := m.AnalyseConfigMap() + if err != nil { + return err + } + return m.ApplyResourceDiff() + +} + +// Update Unified Storage CRD every internal seconds +func (urm *UnifiedResourceManager) RecordUnifiedResources() { + // Get Unified Storage Object +} diff --git a/pkg/manager/memory/memory.go b/pkg/manager/memory/memory.go new file mode 100644 index 0000000..721c058 --- /dev/null +++ b/pkg/manager/memory/memory.go @@ -0,0 +1,114 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memory + +import ( + "io/ioutil" + "os" + + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/utils" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +// ResourceManager ... +type ResourceManager struct { + Memory []*MConfig + pmem utils.Pmemer + configPath string +} + +// NewResourceManager ... +func NewResourceManager() *ResourceManager { + return &ResourceManager{ + Memory: []*MConfig{}, + pmem: utils.NewNodePmemer(), + configPath: "/etc/unified-config/memory", + } +} + +// AnalyseConfigMap analyse memory resource config +func (mrm *ResourceManager) AnalyseConfigMap() error { + memoryConfig := []*MConfig{} + memoryList := &MList{} + + yamlFile, err := ioutil.ReadFile(mrm.configPath) + if err != nil { + if os.IsNotExist(err) { + log.Debugf("memory config file %s not exist", mrm.configPath) + return nil + } + log.Errorf("AnalyseConfigMap:: yamlFile.Get memory error %v", err) + return err + } + err = yaml.Unmarshal(yamlFile, memoryList) + if err != nil { + log.Errorf("AnalyseConfigMap:: parse yaml file error: %v", err) + return err + } + + nodeInfo := config.GlobalConfigVar.NodeInfo + for _, memConfig := range memoryList.Memories { + isMatched := utils.NodeFilter(memConfig.Operator, memConfig.Key, memConfig.Value, nodeInfo) + if isMatched { + conf := &MConfig{} + if len(memConfig.Topology.Regions) != 1 { + log.Errorf("AnalyseConfigMap:: regions has multi config %s", memConfig.Topology.Regions) + continue + } + conf.Region = memConfig.Topology.Regions[0] + conf.Type = memConfig.Topology.Type + memoryConfig = append(memoryConfig, conf) + } + } + mrm.Memory = memoryConfig + return nil +} + +// ApplyResourceDiff apply memory resource to current node +func (mrm *ResourceManager) ApplyResourceDiff() error { + log.Infof("ApplyResourceDiff: matched node resources mrm.Memory: %v", mrm.Memory) + for _, memConfig := range mrm.Memory { + devicePath, _, err := mrm.pmem.GetPmemNamespaceDeivcePath(memConfig.Region, "devdax") + if err != nil { + err := mrm.pmem.CreateNamespace(memConfig.Region, "dax") + if err != nil { + log.Errorf("applyResourceDiff:: create kmem namespace for region [%s], error: %v", memConfig.Region, err) + continue + } + devicePath, _, err = mrm.pmem.GetPmemNamespaceDeivcePath(memConfig.Region, "devdax") + if err != nil { + log.Errorf("applyResourceDiff:: list kmem namespace for region [%s], error: %v", memConfig.Region, err) + continue + } + } + isCreated, err := mrm.pmem.CheckKMEMCreated(devicePath[5:]) + if err != nil { + log.Errorf("applyResourceDiff:: check kmem create error: %v", err) + continue + } + if !isCreated { + err := mrm.pmem.MakeNamespaceMemory(devicePath[5:]) + if err != nil { + log.Errorf("applyRegionQuotaPath:: make kmem memory failed %v", err) + continue + } + } + } + return nil +} diff --git a/pkg/manager/memory/memory_test.go b/pkg/manager/memory/memory_test.go new file mode 100644 index 0000000..7149e38 --- /dev/null +++ b/pkg/manager/memory/memory_test.go @@ -0,0 +1,127 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memory + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/golang/mock/gomock" + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/model" + "github.com/openyurtio/node-resource-manager/pkg/utils" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func makeValidResourceYaml() *model.ResourceYaml { + return &model.ResourceYaml{ + Name: "foo", + } +} + +func makeResourceYamlCustom(tweaks ...func(*model.ResourceYaml)) *model.ResourceYaml { + resourceYaml := makeValidResourceYaml() + for _, fn := range tweaks { + fn(resourceYaml) + } + return resourceYaml +} + +// EnsureVolumeGroupEnv ... +func EnsureVolumeGroupEnv() (string, error, *ResourceManager) { + config.GlobalConfigVar.NodeInfo = &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{"bar": "foo"}, + }, + } + configPath := "/tmp/memory" + mdkirCmd := "mkdir" + _, err := exec.LookPath(mdkirCmd) + if err != nil { + if err == exec.ErrNotFound { + return "", fmt.Errorf("EnsureFolder:: %q executable not found in $PATH", mdkirCmd), nil + } + return "", err, nil + } + + mkdirArgs := []string{"-p", filepath.Dir(configPath)} + //log.Infof("mkdir for folder, the command is %s %v", mdkirCmd, mkdirArgs) + _, err = exec.Command(mdkirCmd, mkdirArgs...).CombinedOutput() + if err != nil { + return "", fmt.Errorf("EnsureFolder:: mkdir for folder error: %v", err), nil + } + + newMockVolumegroupResourceManager := func() *ResourceManager { + return &ResourceManager{ + configPath: configPath, + } + } + return configPath, nil, newMockVolumegroupResourceManager() +} + +func TestAnalyseConfigMap(t *testing.T) { + mockCtl := gomock.NewController(t) + defer mockCtl.Finish() + configPath, err, resourceManager := EnsureVolumeGroupEnv() + if err != nil { + t.Fatal(err) + } + defer os.Remove(configPath) + mockPmemer := utils.NewMockPmemer(mockCtl) + resourceManager.pmem = mockPmemer + setOpInOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpIn + m.Value = "foo" + } + + setPmemTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "pmem", + Regions: []string{"region0"}, + } + } + testYamls := MList{Memories: []model.ResourceYaml{ + *makeResourceYamlCustom(setOpInOperatorElement, setPmemTopology), + }} + + d, err := yaml.Marshal(&testYamls) + if err != nil { + t.Error() + } + err = ioutil.WriteFile(configPath, d, 0777) + if err != nil { + t.Fatal(err) + } + + assert.Nil(t, resourceManager.AnalyseConfigMap()) + assert.Equal(t, 1, len(resourceManager.Memory)) + gomock.InOrder( + mockPmemer.EXPECT().GetPmemNamespaceDeivcePath(gomock.Eq("region0"), gomock.Eq("devdax")).Return("/dev/dax0.0", "", nil), + mockPmemer.EXPECT().CheckKMEMCreated(gomock.Eq("dax0.0")).Return(true, nil), + ) + assert.Nil(t, resourceManager.ApplyResourceDiff()) + +} diff --git a/pkg/manager/memory/type.go b/pkg/manager/memory/type.go new file mode 100644 index 0000000..a1908af --- /dev/null +++ b/pkg/manager/memory/type.go @@ -0,0 +1,32 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package memory + +import ( + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +// MConfig ... +type MConfig struct { + Type string + Region string +} + +// MList ... +type MList struct { + Memories []model.ResourceYaml `yaml:"memory,omitempty"` +} diff --git a/pkg/manager/quotapath/quotapath.go b/pkg/manager/quotapath/quotapath.go new file mode 100644 index 0000000..4f21c86 --- /dev/null +++ b/pkg/manager/quotapath/quotapath.go @@ -0,0 +1,175 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quotapath + +import ( + "io/ioutil" + "os" + "strings" + + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/utils" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" +) + +// ResourceManager ... +type ResourceManager struct { + DeviceQuotaPath map[string]*QpConfig + RegionQuotaPath map[string]*QpConfig + mounter utils.Mounter + mkfsOption []string + pmemer utils.Pmemer + configPath string +} + +// NewResourceManager ... +func NewResourceManager() *ResourceManager { + return &ResourceManager{ + DeviceQuotaPath: make(map[string]*QpConfig), + RegionQuotaPath: make(map[string]*QpConfig), + mounter: utils.NewMounter(), + pmemer: utils.NewNodePmemer(), + configPath: "/etc/unified-config/quotapath", + } +} + +// AnalyseConfigMap analyse quotapath resource config +func (qrm *ResourceManager) AnalyseConfigMap() error { + deviceQuotaConfig := map[string]*QpConfig{} + regionQuotaConfig := map[string]*QpConfig{} + quotaPathList := &QPList{} + yamlFile, err := ioutil.ReadFile(qrm.configPath) + if err != nil { + if os.IsNotExist(err) { + log.Debugf("quota config file %s not exist", qrm.configPath) + return nil + } + log.Errorf("AnalyseConfigMap:: yamlFile.Get error %v", err) + return err + } + err = yaml.Unmarshal(yamlFile, quotaPathList) + if err != nil { + log.Errorf("AnalyseConfigMap:: parse yaml file error: %v", err) + return err + } + mountPathMap := map[string]string{} + nodeInfo := config.GlobalConfigVar.NodeInfo + for _, quotaConfig := range quotaPathList.QuotaPaths { + isMatched := utils.NodeFilter(quotaConfig.Operator, quotaConfig.Key, quotaConfig.Value, nodeInfo) + if isMatched { + if _, ok := mountPathMap[quotaConfig.Name]; ok { + log.Errorf("AnalyseConfigMap:: quotapath config has multi mount path config [%s] on same node", quotaConfig.Name) + continue + } + switch quotaConfig.Topology.Type { + case "device": + conf := &QpConfig{} + if len(quotaConfig.Topology.Devices) != 1 { + log.Errorf("AnalyseConfigMap:: quotapath regions [%v] config only support one device", quotaConfig.Topology.Regions) + continue + } + conf.Device = quotaConfig.Topology.Devices[0] + conf.Fstype = quotaConfig.Topology.Fstype + conf.Options = quotaConfig.Topology.Options + conf.Type = quotaConfig.Topology.Type + deviceQuotaConfig[quotaConfig.Name] = conf + case "pmem": + conf := &QpConfig{} + if len(quotaConfig.Topology.Regions) != 1 { + log.Errorf("AnalyseConfigMap:: quotapath regions [%s] config only support one device", quotaConfig.Topology.Regions) + continue + } + conf.Region = quotaConfig.Topology.Regions[0] + conf.Fstype = quotaConfig.Topology.Fstype + conf.Options = quotaConfig.Topology.Options + conf.Type = quotaConfig.Topology.Type + regionQuotaConfig[quotaConfig.Name] = conf + default: + log.Errorf("AnalyseConfigMap:: not support quotapath config type: [%v]", quotaConfig.Topology.Type) + continue + } + mountPathMap[quotaConfig.Name] = "" + } + } + + qrm.DeviceQuotaPath = deviceQuotaConfig + qrm.RegionQuotaPath = regionQuotaConfig + return nil +} + +// ApplyResourceDiff apply quotapath resource to current node +func (qrm *ResourceManager) ApplyResourceDiff() error { + log.Infof("ApplyResourceDiff: matched node resources qrm.DeviceQuotaPath: %v, qrm.RegionQuotaPath: %v", qrm.DeviceQuotaPath, qrm.RegionQuotaPath) + qrm.mkfsOption = strings.Split("-O project,quota", " ") + err := qrm.applyDeivceQuotaPath() + if err != nil { + log.Errorf("ApplyResourceDiff:: apply deivce quotapath error: %v", err) + } + err = qrm.applyRegionQuotaPath() + if err != nil { + log.Errorf("ApplyResourceDiff:: apply region quotapath error: %v", err) + } + return err +} + +func (qrm *ResourceManager) applyDeivceQuotaPath() error { + + for mountPath, deivceQuotaPathConfig := range qrm.DeviceQuotaPath { + err := qrm.mounter.EnsureFolder(mountPath) + if err != nil { + log.Errorf("applyDeivceQuotaPath:: ensure quotapath error: %v", err) + continue + } + err = qrm.mounter.FormatAndMount(deivceQuotaPathConfig.Device, mountPath, deivceQuotaPathConfig.Fstype, qrm.mkfsOption, deivceQuotaPathConfig.Options) + if err != nil { + log.Errorf("applyDeivceQuotaPath:: mounter FormatAndMount error: %v", err) + continue + } + } + return nil +} + +func (qrm *ResourceManager) applyRegionQuotaPath() error { + for mountPath, regionQuotaPathConfig := range qrm.RegionQuotaPath { + devicePath, _, err := qrm.pmemer.GetPmemNamespaceDeivcePath(regionQuotaPathConfig.Region, "fsdax") + if err != nil { + if strings.Contains(err.Error(), "list Namespace for region get 0 or multi namespaces") { + err := qrm.pmemer.CreateNamespace(regionQuotaPathConfig.Region, "lvm") + if err != nil { + log.Errorf("applyRegionQuotaPath:: create namespace for region [%s], error: %v", regionQuotaPathConfig.Region, err) + continue + } + devicePath, _, err = qrm.pmemer.GetPmemNamespaceDeivcePath(regionQuotaPathConfig.Region, "fsdax") + } else { + log.Errorf("applyRegionQuotaPath:: get region [%s] namespace device path error: %v", regionQuotaPathConfig.Region, err) + continue + } + } + err = qrm.mounter.EnsureFolder(mountPath) + if err != nil { + log.Errorf("applyRegionQuotaPath:: ensure quotapath error: %v", err) + continue + } + err = qrm.mounter.FormatAndMount(devicePath, mountPath, regionQuotaPathConfig.Fstype, qrm.mkfsOption, regionQuotaPathConfig.Options) + if err != nil { + log.Errorf("applyRegionQuotaPath:: mounter FormatAndMount error: %v", err) + continue + } + } + return nil +} diff --git a/pkg/manager/quotapath/quotapath_test.go b/pkg/manager/quotapath/quotapath_test.go new file mode 100644 index 0000000..03bac62 --- /dev/null +++ b/pkg/manager/quotapath/quotapath_test.go @@ -0,0 +1,151 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quotapath + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" + + "github.com/golang/mock/gomock" + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/model" + "github.com/openyurtio/node-resource-manager/pkg/utils" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func makeValidResourceYaml() *model.ResourceYaml { + return &model.ResourceYaml{ + Name: "/tmp/foo", + } +} + +func makeResourceYamlCustom(tweaks ...func(*model.ResourceYaml)) *model.ResourceYaml { + resourceYaml := makeValidResourceYaml() + for _, fn := range tweaks { + fn(resourceYaml) + } + return resourceYaml +} + +// EnsureVolumeGroupEnv ... +func EnsureVolumeGroupEnv() (string, error, *ResourceManager) { + config.GlobalConfigVar.NodeInfo = &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{"bar": "foo"}, + }, + } + configPath := "/tmp/quotapath" + mdkirCmd := "mkdir" + _, err := exec.LookPath(mdkirCmd) + if err != nil { + if err == exec.ErrNotFound { + return "", fmt.Errorf("EnsureFolder:: %q executable not found in $PATH", mdkirCmd), nil + } + return "", err, nil + } + + mkdirArgs := []string{"-p", filepath.Dir(configPath)} + //log.Infof("mkdir for folder, the command is %s %v", mdkirCmd, mkdirArgs) + _, err = exec.Command(mdkirCmd, mkdirArgs...).CombinedOutput() + if err != nil { + return "", fmt.Errorf("EnsureFolder:: mkdir for folder error: %v", err), nil + } + + newMockVolumegroupResourceManager := func() *ResourceManager { + return &ResourceManager{ + configPath: configPath, + } + } + return configPath, nil, newMockVolumegroupResourceManager() +} + +func TestAnalyseDiff(t *testing.T) { + mockCtl := gomock.NewController(t) + defer mockCtl.Finish() + configPath, err, resourceManager := EnsureVolumeGroupEnv() + if err != nil { + t.Fatal(err) + } + defer os.Remove(configPath) + mockPmemer := utils.NewMockPmemer(mockCtl) + mockMounter := utils.NewMockMounter(mockCtl) + resourceManager.mounter = mockMounter + resourceManager.pmemer = mockPmemer + setOpInOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpIn + m.Value = "foo" + } + + setPmemTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "pmem", + Options: "prjquota,shared", + Fstype: "ext4", + Regions: []string{"region0"}, + } + } + setDeviceTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "device", + Options: "prjquota", + Fstype: "ext4", + Devices: []string{"/dev/vdc"}, + } + } + setDeviceNameElement := func(m *model.ResourceYaml) { + m.Name = "/tmp/foo1" + } + testYamls := QPList{QuotaPaths: []model.ResourceYaml{ + *makeResourceYamlCustom(setOpInOperatorElement, setPmemTopology), + *makeResourceYamlCustom(setDeviceNameElement, setOpInOperatorElement, setDeviceTopology), + }} + + d, err := yaml.Marshal(&testYamls) + if err != nil { + t.Error() + } + err = ioutil.WriteFile(configPath, d, 0777) + if err != nil { + t.Fatal(err) + } + + assert.Nil(t, resourceManager.AnalyseConfigMap()) + assert.Equal(t, 1, len(resourceManager.RegionQuotaPath)) + assert.Equal(t, 1, len(resourceManager.DeviceQuotaPath)) + gomock.InOrder( + mockMounter.EXPECT().EnsureFolder( + gomock.Eq("/tmp/foo1")).Return(nil), + mockMounter.EXPECT().FormatAndMount( + gomock.Eq("/dev/vdc"), gomock.Eq("/tmp/foo1"), gomock.Eq("ext4"), gomock.Eq([]string{"-O", "project,quota"}), gomock.Eq("prjquota")).Return(nil), + mockPmemer.EXPECT().GetPmemNamespaceDeivcePath( + gomock.Eq("region0"), gomock.Eq("fsdax")).Return("/dev/pmem0", "", nil), + mockMounter.EXPECT().EnsureFolder( + gomock.Eq("/tmp/foo")).Return(nil), + mockMounter.EXPECT().FormatAndMount( + gomock.Eq("/dev/pmem0"), gomock.Eq("/tmp/foo"), gomock.Eq("ext4"), gomock.Eq([]string{"-O", "project,quota"}), gomock.Eq("prjquota,shared")).Return(nil), + ) + assert.Nil(t, resourceManager.ApplyResourceDiff()) +} diff --git a/pkg/manager/quotapath/type.go b/pkg/manager/quotapath/type.go new file mode 100644 index 0000000..72c791a --- /dev/null +++ b/pkg/manager/quotapath/type.go @@ -0,0 +1,35 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package quotapath + +import ( + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +// QpConfig ... +type QpConfig struct { + Type string + Options string + Fstype string + Region string + Device string +} + +// QPList ... +type QPList struct { + QuotaPaths []model.ResourceYaml `yaml:"quotapath,omitempty"` +} diff --git a/pkg/manager/volumegroup/recorder.go b/pkg/manager/volumegroup/recorder.go new file mode 100644 index 0000000..b2bd271 --- /dev/null +++ b/pkg/manager/volumegroup/recorder.go @@ -0,0 +1,17 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumegroup diff --git a/pkg/manager/volumegroup/type.go b/pkg/manager/volumegroup/type.go new file mode 100644 index 0000000..a3d5dce --- /dev/null +++ b/pkg/manager/volumegroup/type.go @@ -0,0 +1,32 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumegroup + +import ( + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +// VgDeviceConfig ... +type VgDeviceConfig struct { + Name string `json:"name,omitempty"` + PhysicalVolumes []string `json:"physicalVolumes,omitempty"` +} + +// VgList ... +type VgList struct { + VolumeGroups []model.ResourceYaml `yaml:"volumegroup,omitempty"` +} diff --git a/pkg/manager/volumegroup/utils.go b/pkg/manager/volumegroup/utils.go new file mode 100644 index 0000000..0b212f2 --- /dev/null +++ b/pkg/manager/volumegroup/utils.go @@ -0,0 +1,270 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumegroup + +import ( + "crypto/aes" + "crypto/cipher" + "encoding/base64" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "path/filepath" + "strings" + "time" + + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/openyurtio/node-resource-manager/pkg/utils" + log "github.com/sirupsen/logrus" +) + +const ( + // MetadataURL is metadata server url + MetadataURL = "http://100.100.100.200/latest/meta-data/" + // InstanceID is the instance id tag + InstanceID = "instance-id" + // RegionIDTag is the region id tag + RegionIDTag = "region-id" + // NsenterCmd is the nsenter command + NsenterCmd = "/usr/bin/nsenter --mount=/proc/1/ns/mnt " +) + +const ( + // ConfigPath the secret mount file + ConfigPath = "/var/addon/token-config" +) + +// AKInfo access key info +type AKInfo struct { + // AccessKeyId access key id + AccessKeyID string `json:"access.key.id"` + // AccessKeySecret access key secret + AccessKeySecret string `json:"access.key.secret"` + // SecurityToken security token + SecurityToken string `json:"security.token"` + // Expiration expiration duration + Expiration string `json:"expiration"` + // Keyring key ring + Keyring string `json:"keyring"` +} + +// RoleAuth define STS Token Response +type RoleAuth struct { + AccessKeyID string + AccessKeySecret string + Expiration time.Time + SecurityToken string + LastUpdated time.Time + Code string +} + +func ListDevice(vgName string) []string { + cmd := fmt.Sprintf("%s pvscan | grep \"VG %s\" | awk '{print $2}'", NsenterCmd, vgName) + out, err := utils.Run(cmd) + if err != nil { + devs := make([]string, 0) + return devs + } + + outLines := strings.Split(out, "\n") + deviceList := []string{} + for _, line := range outLines { + line = strings.TrimSpace(line) + if line != "" && !strings.HasPrefix(line, "WARNING") { + deviceList = append(deviceList, line) + } + } + return deviceList +} + +func diffDevice(devList1, devList2 []string) bool { + if len(devList1) != len(devList2) { + return true + } + for _, dev1 := range devList1 { + isSearched := false + for _, dev2 := range devList2 { + if dev1 == dev2 { + isSearched = true + break + } + } + if !isSearched { + return true + } + } + + return false +} + +// NewEcsClient create a ecsClient object +func NewEcsClient(accessKeyID, accessKeySecret, accessToken string) (ecsClient *ecs.Client) { + var err error + if accessToken == "" { + ecsClient, err = ecs.NewClientWithAccessKey("cn-hangzhou", accessKeyID, accessKeySecret) + if err != nil { + return nil + } + } else { + ecsClient, err = ecs.NewClientWithStsToken("cn-hangzhou", accessKeyID, accessKeySecret, accessToken) + if err != nil { + return nil + } + } + return +} + +// GetMetaData get host regionid, zoneid +func GetMetaData(resource string) string { + resp, err := http.Get(MetadataURL + resource) + if err != nil { + return "" + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "" + } + return string(body) +} + +// GetDefaultAK read default ak from local file or from STS +func GetDefaultAK() (string, string, string) { + accessKeyID, accessSecret := GetLocalAK() + + accessToken := "" + if accessKeyID == "" || accessSecret == "" { + accessKeyID, accessSecret, accessToken = GetManagedToken() + if accessKeyID != "" { + log.Infof("Get AK: use Managed AK") + } + } else { + log.Infof("Get AK: use Local AK") + } + + if accessKeyID == "" || accessSecret == "" { + accessKeyID, accessSecret, accessToken = GetSTSAK() + } + + return accessKeyID, accessSecret, accessToken +} + +// GetLocalAK read ossfs ak from local or from secret file +func GetLocalAK() (string, string) { + accessKeyID, accessSecret := "", "" + accessKeyID = os.Getenv("ACCESS_KEY_ID") + accessSecret = os.Getenv("ACCESS_KEY_SECRET") + + return strings.TrimSpace(accessKeyID), strings.TrimSpace(accessSecret) +} + +// GetSTSAK get STS AK and token from ecs meta server +func GetSTSAK() (string, string, string) { + roleAuth := RoleAuth{} + subpath := "ram/security-credentials/" + roleName := GetMetaData(subpath) + fullPath := filepath.Join(subpath, roleName) + roleInfo := GetMetaData(fullPath) + + err := json.Unmarshal([]byte(roleInfo), &roleAuth) + if err != nil { + log.Errorf("GetSTSToken: unmarshal roleInfo: %s, with error: %s", roleInfo, err.Error()) + return "", "", "" + } + return roleAuth.AccessKeyID, roleAuth.AccessKeySecret, roleAuth.SecurityToken +} + +// GetManagedToken get ak from csi secret +func GetManagedToken() (string, string, string) { + var akInfo AKInfo + AccessKeyID, AccessKeySecret, SecurityToken := "", "", "" + if _, err := os.Stat(ConfigPath); err == nil { + encodeTokenCfg, err := ioutil.ReadFile(ConfigPath) + if err != nil { + log.Errorf("failed to read token config, err: %v", err) + return "", "", "" + } + err = json.Unmarshal(encodeTokenCfg, &akInfo) + if err != nil { + log.Errorf("error unmarshal token config: %v", err) + return "", "", "" + } + keyring := akInfo.Keyring + ak, err := Decrypt(akInfo.AccessKeyID, []byte(keyring)) + if err != nil { + log.Errorf("failed to decode ak, err: %v", err) + return "", "", "" + } + + sk, err := Decrypt(akInfo.AccessKeySecret, []byte(keyring)) + if err != nil { + log.Errorf("failed to decode sk, err: %v", err) + return "", "", "" + } + + token, err := Decrypt(akInfo.SecurityToken, []byte(keyring)) + if err != nil { + log.Errorf("failed to decode token, err: %v", err) + return "", "", "" + } + layout := "2006-01-02T15:04:05Z" + t, err := time.Parse(layout, akInfo.Expiration) + if err != nil { + log.Errorf("Parse expiration error: %s", err.Error()) + } + if t.Before(time.Now()) { + log.Errorf("invalid token which is expired, expiration as: %s", akInfo.Expiration) + } + AccessKeyID = string(ak) + AccessKeySecret = string(sk) + SecurityToken = string(token) + } + return AccessKeyID, AccessKeySecret, SecurityToken +} + +// PKCS5UnPadding get pkc +func PKCS5UnPadding(origData []byte) []byte { + length := len(origData) + unpadding := int(origData[length-1]) + return origData[:(length - unpadding)] +} + +// Decrypt secret Decrypt +func Decrypt(s string, keyring []byte) ([]byte, error) { + cdata, err := base64.StdEncoding.DecodeString(s) + if err != nil { + log.Errorf("failed to decode base64 string, err: %v", err) + return nil, err + } + block, err := aes.NewCipher(keyring) + if err != nil { + log.Errorf("failed to new cipher, err: %v", err) + return nil, err + } + blockSize := block.BlockSize() + + iv := cdata[:blockSize] + blockMode := cipher.NewCBCDecrypter(block, iv) + origData := make([]byte, len(cdata)-blockSize) + + blockMode.CryptBlocks(origData, cdata[blockSize:]) + + origData = PKCS5UnPadding(origData) + return origData, nil +} diff --git a/pkg/manager/volumegroup/volumegroup.go b/pkg/manager/volumegroup/volumegroup.go new file mode 100644 index 0000000..f1d1c6b --- /dev/null +++ b/pkg/manager/volumegroup/volumegroup.go @@ -0,0 +1,432 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumegroup + +import ( + "errors" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/aliyun/alibaba-cloud-sdk-go/services/ecs" + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/utils" + log "github.com/sirupsen/logrus" + yaml "gopkg.in/yaml.v2" +) + +const ( + StorageCmNameSpace = "kube-system" + StorageCmName = "cm-node-resource" + VGConfigKey = "volumegroup.json" + AliyunLocalDisk = "aliyun-local-disk" + + VgConfigFile = "/etc/unified-config/volumegroup" + VgTypeDevice = "device" + VgTypePvc = "pvc" + VgTypeLocal = "alibabacloud-local-disk" + VgTypePmem = "pmem" +) + +// ResourceManager ... +type ResourceManager struct { + volumeGroupDeviceMap map[string]*VgDeviceConfig + volumeGroupRegionMap map[string][]string + pmemer utils.Pmemer + lvmer utils.LVM + configPath string +} + +// NewResourceManager ... +func NewResourceManager() *ResourceManager { + return &ResourceManager{ + volumeGroupDeviceMap: make(map[string]*VgDeviceConfig), + volumeGroupRegionMap: make(map[string][]string), + pmemer: utils.NewNodePmemer(), + lvmer: utils.NewNodeLVM(), + configPath: "/etc/unified-config/volumegroup", + } +} + +// DeviceChars ... +var DeviceChars = []string{"b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "w", "x", "y", "z"} + +// Create VolumeGroup +func (vrm *ResourceManager) createVg(vgName string, desirePvList []string) error { + pvListStr := strings.Join(desirePvList, " ") + tagList := []string{} + out, err := vrm.lvmer.CreateVG(vgName, pvListStr, tagList) + if err != nil { + log.Errorf("createVg:: Create Vg(%s) error with: %s", vgName, err.Error()) + return err + } + log.Infof("createVg:: Successful Create Vg(%s) with out: %s", vgName, out) + return nil +} + +// Upgrade VolumeGroup, only support extend pv; +func (vrm *ResourceManager) updateVg(vgName string, expectPvList, realPvList []string) error { + if len(expectPvList) < len(realPvList) { + msg := fmt.Sprintf("updateVg:: VolumeGroup: %s, expected pv list should be more than current pv list when update volume group: %v, %v", vgName, expectPvList, realPvList) + log.Errorf(msg) + return errors.New(msg) + } + + // removed pv: pv exist in current node, but not in expect + removePv := difference(realPvList, expectPvList) + if len(removePv) > 0 { + msg := fmt.Sprintf("updateVg:: VolumeGroup: %s, expected pv list should be more than current pv list when update volume group: expect %v, current %v, and not support remove pv now", vgName, expectPvList, realPvList) + log.Errorf(msg) + return errors.New(msg) + } + + // added pv: pv exist in expect, but not in current node. + addedPv := difference(expectPvList, realPvList) + if len(addedPv) == 0 { + msg := fmt.Sprintf("updateVg:: VolumeGroup: %s, expected pv list same with current pv list, expect %v, current %v", vgName, expectPvList, realPvList) + log.Errorf(msg) + return errors.New(msg) + } + + pvListStr := strings.Join(addedPv, " ") + _, err := vrm.lvmer.ExtendVG(vgName, pvListStr) + if err != nil { + msg := fmt.Sprintf("updateVg:: Extend vg(%s) error: %v", vgName, err) + log.Errorf(msg) + return errors.New(msg) + } + log.Infof("updateVg:: Successful Add pvs %s to VolumeGroup %s", addedPv, vgName) + return nil +} + +// difference returns the elements in `a` that aren't in `b`. +func difference(a, b []string) []string { + mb := make(map[string]struct{}, len(b)) + for _, x := range b { + mb[x] = struct{}{} + } + var diff []string + for _, x := range a { + if _, found := mb[x]; !found { + diff = append(diff, x) + } + } + return diff +} + +func getPvListForLocalDisk() []string { + // Step 2: Get LocalDisk Number + localDeviceList := []string{} + localDeviceNum, err := getLocalDeviceNum() + if err != nil { + log.Errorf("getPvListForLocalDisk:: LocalDiskMount:: Get Local Disk Number Error, Error: %s", err.Error()) + return localDeviceList + } + if localDeviceNum < 1 { + log.Errorf("getPvListForLocalDisk:: VG not exist and also not local disk exist, localDeivceNum: %v", localDeviceNum) + return localDeviceList + } + + // Step 3: Get LocalDisk device + deviceStartWith := "vdb" + deviceNamePrefix := "vd" + deviceStartIndex := 0 + deviceNameLen := len(deviceStartWith) + if deviceNameLen > 1 { + deviceStartChar := deviceStartWith[deviceNameLen-1 : deviceNameLen] + for index := 0; index < 15; index++ { + if deviceStartChar == DeviceChars[index] { + deviceStartIndex = index + } + } + deviceNamePrefix = deviceStartWith[0 : deviceNameLen-1] + } + for i := deviceStartIndex; i < localDeviceNum; i++ { + deviceName := deviceNamePrefix + DeviceChars[i] + devicePath := filepath.Join("/dev", deviceName) + localDeviceList = append(localDeviceList, devicePath) + } + //log.Infof("doLocalVolumeMounts, Starting LocalDisk Mount: LocalDisk Number: %d, LocalDisk: %v", localDeviceNum, localDeviceList) + return localDeviceList +} + +// Get Local Disk Number from ecs API +// Requirements: The instance must have role which contains ecs::DescribeInstances, ecs::DescribeInstancesType. +func getLocalDeviceNum() (int, error) { + instanceID := GetMetaData(InstanceID) + regionID := GetMetaData(RegionIDTag) + localDeviceNum := 0 + akID, akSecret, token := GetDefaultAK() + client := NewEcsClient(akID, akSecret, token) + + // Get Instance Type + request := ecs.CreateDescribeInstancesRequest() + request.RegionId = regionID + request.InstanceIds = "[\"" + instanceID + "\"]" + instanceResponse, err := client.DescribeInstances(request) + if err != nil { + log.Errorf("getLocalDeviceNum:: Describe Instance: %s Error: %s", instanceID, err.Error()) + return -1, err + } + if instanceResponse == nil || len(instanceResponse.Instances.Instance) == 0 { + log.Infof("getLocalDeviceNum:: Describe Instance Error, with empty response: %s", instanceID) + return -1, err + } + + // Get Instance LocalDisk Number + instanceTypeID := instanceResponse.Instances.Instance[0].InstanceType + instanceTypeFamily := instanceResponse.Instances.Instance[0].InstanceTypeFamily + instanceTypeRequest := ecs.CreateDescribeInstanceTypesRequest() + instanceTypeRequest.InstanceTypeFamily = instanceTypeFamily + response, err := client.DescribeInstanceTypes(instanceTypeRequest) + if err != nil { + log.Errorf("getLocalDeviceNum:: Describe Instance: %s, Type: %s, Family: %s Error: %s", instanceID, instanceTypeID, instanceTypeFamily, err.Error()) + return -1, err + } + for _, instance := range response.InstanceTypes.InstanceType { + if instance.InstanceTypeId == instanceTypeID { + localDeviceNum = instance.LocalStorageAmount + log.Infof("getLocalDeviceNum:: Instance: %s, InstanceType: %s, InstanceLocalDiskNum: %d", instanceID, instanceTypeID, localDeviceNum) + break + } + } + return localDeviceNum, nil +} + +// Get current VolumeGroup in node +// echo vgOjbect contains: vgName, pvList; +func (vrm *ResourceManager) getRealVgList() ([]*VgDeviceConfig, error) { + physicalVolumeList, err := vrm.lvmer.ListPhysicalVolume() + if err != nil { + log.Errorf("List PhysicalVolume get error %v", err) + return nil, err + } + log.Debugf("Real VolumeGroup List: %+v", physicalVolumeList) + nodeVgList := []*VgDeviceConfig{} + for _, physicalVolume := range physicalVolumeList { + isAlreadyAdded := false + for _, nodeVg := range nodeVgList { + if nodeVg.Name == physicalVolume.VgName { + nodeVg.PhysicalVolumes = append(nodeVg.PhysicalVolumes, physicalVolume.Name) + isAlreadyAdded = true + log.Debugf("getRealVgList:: physicalVolumes: %v, physicalVolumeName: %s, volumeGroupName: %v", nodeVg.PhysicalVolumes, physicalVolume.Name, physicalVolume.VgName) + } + } + if isAlreadyAdded == false { + vgPvConfig := &VgDeviceConfig{} + vgPvConfig.Name = physicalVolume.VgName + vgPvConfig.PhysicalVolumes = append(vgPvConfig.PhysicalVolumes, physicalVolume.Name) + nodeVgList = append(nodeVgList, vgPvConfig) + log.Debugf("Add New VolumeGroupPvConfig: %s, %s, %v", physicalVolume.Name, physicalVolume.VgName, vgPvConfig) + } + } + return nodeVgList, nil +} + +// AnalyseConfigMap analyse pmem resource config +func (vrm *ResourceManager) AnalyseConfigMap() error { + + vgDeviceMap := map[string]*VgDeviceConfig{} + vgRegionMap := map[string][]string{} + + volumeGroupList := &VgList{} + yamlFile, err := ioutil.ReadFile(vrm.configPath) + if err != nil { + if os.IsNotExist(err) { + log.Debugf("volume config file %s not exist", vrm.configPath) + return nil + } + log.Errorf("AnalyseConfigMap:: ReadFile: yamlFile.Get error #%v ", err) + return err + } + + err = yaml.Unmarshal(yamlFile, volumeGroupList) + if err != nil { + log.Errorf("AnalyseConfigMap:: Unmarshal: parse yaml file error: %v", err) + return err + } + + nodeInfo := config.GlobalConfigVar.NodeInfo + for _, devConfig := range volumeGroupList.VolumeGroups { + vgDeviceConfig := &VgDeviceConfig{} + + isMatched := utils.NodeFilter(devConfig.Operator, devConfig.Key, devConfig.Value, nodeInfo) + log.Infof("AnalyseConfigMap:: isMatched: %v, devConfig: %+v", isMatched, devConfig) + + if isMatched { + switch devConfig.Topology.Type { + case VgTypeDevice: + vgDeviceConfig.PhysicalVolumes = devConfig.Topology.Devices + vgDeviceMap[devConfig.Name] = vgDeviceConfig + case VgTypeLocal: + tmpConfig := &VgDeviceConfig{} + tmpConfig.PhysicalVolumes = getPvListForLocalDisk() + vgDeviceMap[devConfig.Name] = tmpConfig + case VgTypePvc: + // not support yet + continue + case VgTypePmem: + vgRegionMap[devConfig.Name] = devConfig.Topology.Regions + default: + log.Errorf("AnalyseConfigMap:: Get unsupported volumegroup type: %s", devConfig.Topology.Type) + continue + } + } + } + vrm.volumeGroupDeviceMap = vgDeviceMap + vrm.volumeGroupRegionMap = vgRegionMap + return nil +} + +// ApplyResourceDiff apply volume group resource to current node +func (vrm *ResourceManager) ApplyResourceDiff() error { + + // Get Actual VolumeGroup on node. + actualVgConfig, err := vrm.getRealVgList() + if err != nil { + log.Errorf("ApplyResourceDiff:: Get Node Actual VolumeGroup Error: %s", err.Error()) + return err + } + if len(vrm.volumeGroupDeviceMap) > 0 { + vrm.applyDeivce(actualVgConfig) + } + if len(vrm.volumeGroupRegionMap) > 0 { + vrm.applyRegion(actualVgConfig) + } + + log.Infof("ApplyResourceDiff:: Finish volumegroup loop...") + return nil +} + +func (vrm *ResourceManager) applyDeivce(actualVgConfig []*VgDeviceConfig) error { + // process each expect volume group + for expectVgName, expectVg := range vrm.volumeGroupDeviceMap { + log.Infof("applyDevice:: expectName: %s, expectVgDevices: %v", expectVgName, expectVg.PhysicalVolumes) + isVgExist := false + isVgNeedUpdate := false + realPhysicalVolumeList := []string{} + + for _, realVg := range actualVgConfig { + if expectVgName == realVg.Name { + isVgExist = true + diffs := difference(expectVg.PhysicalVolumes, realVg.PhysicalVolumes) + if len(diffs) != 0 { + realPhysicalVolumeList = realVg.PhysicalVolumes + isVgNeedUpdate = true + } + break + } + } + if !isVgExist { + log.Infof("Create VolumeGroup:: %+v, %+v", expectVgName, expectVg.PhysicalVolumes) + return vrm.createVg(expectVgName, expectVg.PhysicalVolumes) + } else if isVgNeedUpdate { + log.Infof("Update VolumeGroup:: %+v, %+v", expectVgName, expectVg.PhysicalVolumes) + return vrm.updateVg(expectVgName, expectVg.PhysicalVolumes, realPhysicalVolumeList) + } + } + return nil +} + +func (vrm *ResourceManager) applyRegion(actualVgConfig []*VgDeviceConfig) error { + regions, err := vrm.pmemer.GetRegions() + if err != nil { + log.Errorf("applyRegion: get pmem regions error: %v", err) + return err + } + + for _, expectRegions := range vrm.volumeGroupRegionMap { + expectNamespaces := []string{} + for _, expectRegion := range expectRegions { + expectRegionExists := false + for _, region := range regions.Regions { + if expectRegion == region.Dev { + expectRegionExists = true + if len(region.Namespaces) == 0 { + vrm.pmemer.CreateNamespace(region.Dev, "lvm") + } + } + } + if !expectRegionExists { + err := fmt.Errorf("applyRegion:: expect region %s not exists", expectRegion) + return err + } + expectNamespaces = append(expectNamespaces, utils.ConvertRegion2Namespace(expectRegion)) + } + } + updatedRegions, err := vrm.pmemer.GetRegions() + if err != nil { + log.Errorf("applyRegion: get pmem regions error: %v", err) + return err + } + for expectVgName, expectRegions := range vrm.volumeGroupRegionMap { + log.Infof("applyDevice:: expectVgName: %v, expectRegions: %v", expectVgName, expectRegions) + expectLvmInUseDevices := []string{} + expectLvmNotInUseDevices := []string{} + for _, expectRegion := range expectRegions { + devicePath := utils.ConvertNamespace2LVMDevicePath(utils.ConvertRegion2Namespace(expectRegion), updatedRegions) + if devicePath == "" { + log.Errorf("applyRegion:: did not get namespace.Blockdev from expectRegion: %s, regions: %v", expectRegion, updatedRegions) + } + if vrm.pmemer.CheckNamespaceUsed(devicePath) { + log.Warnf("NameSpace heen used region: %v, devicePath: %s", expectRegion, devicePath) + expectLvmInUseDevices = append(expectLvmInUseDevices, devicePath) + } + expectLvmNotInUseDevices = append(expectLvmNotInUseDevices, devicePath) + } + + isVgNeedCreate := true + for _, actualVg := range actualVgConfig { + if expectVgName == actualVg.Name { + isVgNeedCreate = false + if otherUsage := difference(expectLvmInUseDevices, actualVg.PhysicalVolumes); len(otherUsage) != 0 { + log.Errorf("applyRegion:: device [%s] is used in other usage", otherUsage) + break + } + updatePvs := difference(expectLvmNotInUseDevices, actualVg.PhysicalVolumes) + if len(updatePvs) == 0 { + break + } + vrm.updatePmemVg(expectVgName, expectLvmNotInUseDevices) + } + } + if isVgNeedCreate { + if len(expectLvmInUseDevices) != 0 { + log.Errorf("applyRegion:: attempt to use inused devices [%s] to create volumegroup", expectLvmInUseDevices) + continue + } + vrm.createVg(expectVgName, expectLvmNotInUseDevices) + } + } + + return nil +} + +func (vrm *ResourceManager) updatePmemVg(vgName string, addedPv []string) error { + + pvListStr := strings.Join(addedPv, " ") + _, err := vrm.lvmer.ExtendVG(vgName, pvListStr) + if err != nil { + msg := fmt.Sprintf("updatePmemVg:: Extend vg(%s) error: %v", vgName, err) + log.Errorf(msg) + return errors.New(msg) + } + log.Infof("updatePmemVg:: Successful Add pvs %s to VolumeGroup %s", addedPv, vgName) + return nil +} diff --git a/pkg/manager/volumegroup/volumegroup_test.go b/pkg/manager/volumegroup/volumegroup_test.go new file mode 100644 index 0000000..737ab54 --- /dev/null +++ b/pkg/manager/volumegroup/volumegroup_test.go @@ -0,0 +1,200 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volumegroup + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/golang/mock/gomock" + "github.com/openyurtio/node-resource-manager/pkg/config" + "github.com/openyurtio/node-resource-manager/pkg/model" + "github.com/openyurtio/node-resource-manager/pkg/utils" + "github.com/stretchr/testify/assert" + yaml "gopkg.in/yaml.v2" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func makeValidResourceYaml() *model.ResourceYaml { + return &model.ResourceYaml{ + Name: "foo", + } +} + +func makeResourceYamlCustom(tweaks ...func(*model.ResourceYaml)) *model.ResourceYaml { + resourceYaml := makeValidResourceYaml() + for _, fn := range tweaks { + fn(resourceYaml) + } + return resourceYaml +} + +// EnsureVolumeGroupEnv ... +func EnsureVolumeGroupEnv() (string, error, *ResourceManager) { + config.GlobalConfigVar.NodeInfo = &v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "foo", + Labels: map[string]string{"bar": "foo"}, + }, + } + configPath := "/tmp/volumegroup" + err := EnsureFolder(filepath.Dir(configPath)) + if err != nil { + return "", err, nil + } + + newMockVolumegroupResourceManager := func() *ResourceManager { + return &ResourceManager{ + configPath: configPath, + } + } + return configPath, nil, newMockVolumegroupResourceManager() +} + +func TestAnalyseConfigMap(t *testing.T) { + // set test node info + configPath, err, resourceManager := EnsureVolumeGroupEnv() + if err != nil { + t.Fatal(err) + } + defer os.Remove(configPath) + setOpInOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpIn + m.Value = "foo" + } + setOpNotInOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpNotIn + m.Value = "foo" + } + setOpExistsOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpExists + m.Value = "foo" + } + setOpNotExistsOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpExists + m.Value = "foo" + } + setRyNameBar1Element := func(m *model.ResourceYaml) { + m.Name = "bar1" + } + setDeviceTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "device", + Devices: []string{"/dev/vdb", "/dev/vdc"}, + } + } + setPmemTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "pmem", + Regions: []string{"/dev/vdb", "/dev/vdc"}, + } + } + + testYamls := VgList{VolumeGroups: []model.ResourceYaml{ + *makeResourceYamlCustom(setOpInOperatorElement, setDeviceTopology), + *makeResourceYamlCustom(setOpNotInOperatorElement), + *makeResourceYamlCustom(setOpNotExistsOperatorElement), + *makeResourceYamlCustom(setOpExistsOperatorElement, setRyNameBar1Element, setPmemTopology), + }} + d, err := yaml.Marshal(&testYamls) + defer os.Remove(configPath) + if err != nil { + t.Error() + } + ioutil.WriteFile(configPath, d, 0777) + + assert.Nil(t, resourceManager.AnalyseConfigMap()) + assert.Equal(t, 1, len(resourceManager.volumeGroupDeviceMap)) + assert.Equal(t, 1, len(resourceManager.volumeGroupRegionMap)) +} + +// EnsureFolder ... +func EnsureFolder(target string) error { + mdkirCmd := "mkdir" + _, err := exec.LookPath(mdkirCmd) + if err != nil { + if err == exec.ErrNotFound { + return fmt.Errorf("EnsureFolder:: %q executable not found in $PATH", mdkirCmd) + } + return err + } + + mkdirArgs := []string{"-p", target} + //log.Infof("mkdir for folder, the command is %s %v", mdkirCmd, mkdirArgs) + _, err = exec.Command(mdkirCmd, mkdirArgs...).CombinedOutput() + if err != nil { + return fmt.Errorf("EnsureFolder:: mkdir for folder error: %v", err) + } + return nil +} + +func TestAnalyseDiff(t *testing.T) { + mockCtl := gomock.NewController(t) + defer mockCtl.Finish() + configPath, err, resourceManager := EnsureVolumeGroupEnv() + if err != nil { + t.Fatal(err) + } + defer os.Remove(configPath) + mockPmemer := utils.NewMockPmemer(mockCtl) + resourceManager.pmemer = mockPmemer + mockLVM := utils.NewMockLVM(mockCtl) + resourceManager.lvmer = mockLVM + setOpInOperatorElement := func(m *model.ResourceYaml) { + m.Key = "bar" + m.Operator = metav1.LabelSelectorOpIn + m.Value = "foo" + } + setVgDeviceTopology := func(m *model.ResourceYaml) { + m.Topology = model.Topology{ + Type: "device", + Devices: []string{"/dev/vdb", "/dev/vdc"}, + } + } + setVgDeviceName := func(m *model.ResourceYaml) { + m.Name = "volumegroup1" + } + + testYamls := VgList{VolumeGroups: []model.ResourceYaml{ + *makeResourceYamlCustom(setOpInOperatorElement, setVgDeviceTopology, setVgDeviceName), + }} + d, err := yaml.Marshal(&testYamls) + if err != nil { + t.Error() + } + err = ioutil.WriteFile(configPath, d, 0777) + if err != nil { + t.Fatal(err) + } + prListStr := strings.Join([]string{"/dev/vdb", "/dev/vdc"}, " ") + assert.Nil(t, resourceManager.AnalyseConfigMap()) + gomock.InOrder( + mockLVM.EXPECT().ListPhysicalVolume().Return([]*model.PV{}, nil), + mockLVM.EXPECT().CreateVG(gomock.Eq("volumegroup1"), gomock.Eq(prListStr), gomock.Eq([]string{})).Return("", nil), + ) + assert.Nil(t, resourceManager.ApplyResourceDiff()) +} diff --git a/pkg/model/type.go b/pkg/model/type.go new file mode 100644 index 0000000..ac38f86 --- /dev/null +++ b/pkg/model/type.go @@ -0,0 +1,417 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package model + +import ( + "fmt" + "strconv" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// VolumeType is volume type +type VolumeType byte + +const ( + // separator for vg info + separator = "<:SEP:>" +) + +var volumeTypeKeys = []byte("mMoOrRsSpviIlcVtTe") + +// types +const ( + VolumeTypeMirrored VolumeType = 'm' + VolumeTypeMirroredWithoutSync VolumeType = 'M' + VolumeTypeOrigin VolumeType = 'o' + VolumeTypeOriginWithMergingSnapshot VolumeType = 'O' + VolumeTypeRAID VolumeType = 'r' + VolumeTypeRAIDWithoutSync VolumeType = 'R' + VolumeTypeSnapshot VolumeType = 's' + VolumeTypeMergingSnapshot VolumeType = 'S' + VolumeTypePVMove VolumeType = 'p' + VolumeTypeVirtualMirror VolumeType = 'v' + VolumeTypeVirtualRaidImage VolumeType = 'i' + VolumeTypeRaidImageOutOfSync VolumeType = 'I' + VolumeTypeMirrorLog VolumeType = 'l' + VolumeTypeUnderConversion VolumeType = 'c' + VolumeTypeThin VolumeType = 'V' + VolumeTypeThinPool VolumeType = 't' + VolumeTypeThinPoolData VolumeType = 'T' + VolumeTypeRaidOrThinPoolMetadata VolumeType = 'e' +) + +// VolumePermissions is volume permissions +type VolumePermissions rune + +var volumePermissonsKeys = []byte("wrR") + +// permissions +const ( + VolumePermissionsWriteable VolumePermissions = 'w' + VolumePermissionsReadOnly VolumePermissions = 'r' + VolumePermissionsReadOnlyActivation VolumePermissions = 'R' +) + +// VolumeAllocation is volume allocation policy +type VolumeAllocation rune + +var volumeAllocationKeys = []byte("acilnACILN") + +// allocations +const ( + VolumeAllocationAnywhere VolumeAllocation = 'a' + VolumeAllocationContiguous VolumeAllocation = 'c' + VolumeAllocationInherited VolumeAllocation = 'i' + VolumeAllocationCling VolumeAllocation = 'l' + VolumeAllocationNormal VolumeAllocation = 'n' + VolumeAllocationAnywhereLocked VolumeAllocation = 'A' + VolumeAllocationContiguousLocked VolumeAllocation = 'C' + VolumeAllocationInheritedLocked VolumeAllocation = 'I' + VolumeAllocationClingLocked VolumeAllocation = 'L' + VolumeAllocationNormalLocked VolumeAllocation = 'N' +) + +// VolumeFixedMinor is volume fixed minor +type VolumeFixedMinor rune + +// fixed minor +const ( + VolumeFixedMinorEnabled VolumeFixedMinor = 'm' + VolumeFixedMinorDisabled VolumeFixedMinor = '-' +) + +func (t VolumeFixedMinor) toProto() bool { + return t == VolumeFixedMinorEnabled +} + +// VolumeState is volume state +type VolumeState rune + +var volumeStateKeys = []byte("asISmMdi") + +// states +const ( + VolumeStateActive VolumeState = 'a' + VolumeStateSuspended VolumeState = 's' + VolumeStateInvalidSnapshot VolumeState = 'I' + VolumeStateInvalidSuspendedSnapshot VolumeState = 'S' + VolumeStateSnapshotMergeFailed VolumeState = 'm' + VolumeStateSuspendedSnapshotMergeFailed VolumeState = 'M' + VolumeStateMappedDevicePresentWithoutTables VolumeState = 'd' + VolumeStateMappedDevicePresentWithInactiveTable VolumeState = 'i' +) + +// VolumeOpen is volume open +type VolumeOpen rune + +// open +const ( + VolumeOpenIsOpen VolumeOpen = 'o' + VolumeOpenIsNotOpen VolumeOpen = '-' +) + +// VolumeTargetType is volume taget type +type VolumeTargetType rune + +var volumeTargetTypeKeys = []byte("mrstuv") + +// target type +const ( + VolumeTargetTypeMirror VolumeTargetType = 'm' + VolumeTargetTypeRAID VolumeTargetType = 'r' + VolumeTargetTypeSnapshot VolumeTargetType = 's' + VolumeTargetTypeThin VolumeTargetType = 't' + VolumeTargetTypeUnknown VolumeTargetType = 'u' + VolumeTargetTypeVirtual VolumeTargetType = 'v' +) + +// VolumeZeroing is volume zeroing +type VolumeZeroing rune + +// zeroing +const ( + VolumeZeroingIsZeroing VolumeZeroing = 'z' + VolumeZeroingIsNonZeroing VolumeZeroing = '-' +) + +// VolumeHealth is volume health +type VolumeHealth rune + +// health +const ( + VolumeHealthOK VolumeHealth = '-' + VolumeHealthPartial VolumeHealth = 'p' + VolumeHealthRefreshNeeded VolumeHealth = 'r' + VolumeHealthMismatchesExist VolumeHealth = 'm' + VolumeHealthWritemostly VolumeHealth = 'w' +) + +// VolumeActivationSkipped is volume activation +type VolumeActivationSkipped rune + +// activation +const ( + VolumeActivationSkippedIsSkipped VolumeActivationSkipped = 's' + VolumeActivationSkippedIsNotSkipped VolumeActivationSkipped = '-' +) + +func (t VolumeActivationSkipped) toProto() bool { + return t == VolumeActivationSkippedIsSkipped +} + +// ResourceYaml ... +type ResourceYaml struct { + Name string `yaml:"name,omitempty"` + Key string `yaml:"key,omitempty"` + Operator metav1.LabelSelectorOperator `yaml:"operator,omitempty"` + Value string `yaml:"value,omitempty"` + Topology Topology `yaml:"topology,omitempty"` +} + +// Topology ... +type Topology struct { + Type string `yaml:"type,omitempty"` + Options string `yaml:"options,omitempty"` + Fstype string `yaml:"fstype,omitempty"` + + Devices []string `yaml:"devices,omitempty"` + Volumes []map[string]string `yaml:"volumes,omitempty"` + Regions []string `yaml:"regions,omitempty"` +} + +// PmemRegions list all regions +type PmemRegions struct { + Regions []PmemRegion `json:"regions"` +} + +// PmemRegion define on pmem region +type PmemRegion struct { + Dev string `json:"dev"` + Size int64 `json:"size,omitempty"` + AvailableSize int64 `json:"available_size,omitempty"` + MaxAvailableExent int64 `json:"max_available_extent,omitempty"` + RegionType string `json:"type,omitempty"` + IsetID int64 `json:"iset_id,omitempty"` + PersistenceDomain string `json:"persistence_domain,omitempty"` + Namespaces []PmemNameSpace `json:"namespaces,omitempty"` +} + +// PmemNameSpace define one pmem namespaces +type PmemNameSpace struct { + Dev string `json:"dev,omitempty"` + Mode string `json:"mode,omitempty"` + MapType string `json:"map,omitempty"` + Size int64 `json:"size,omitempty"` + UUID string `json:"uuid,omitempty"` + SectorSize int64 `json:"sectorsize,omitempty"` + Align int64 `json:"align,omitempty"` + BlockDev string `json:"blockdev,omitempty"` + CharDev string `json:"chardev,omitempty"` + Name string `json:"name,omitempty"` +} + +// DaxctrlMem list all mems +type DaxctrlMem struct { + Chardev string `json:"chardev"` + Size int64 `json:"size"` + TargetNode int `json:"target_node"` + Mode string `json:"mode"` + Movable bool `json:"movable"` +} + +// LV is a logical volume +type LV struct { + Name string + Size uint64 + UUID string + Attributes LVAttributes + CopyPercent string + ActualDevMajNumber uint32 + ActualDevMinNumber uint32 + Tags []string +} + +// VG is volume group +type VG struct { + Name string + Size uint64 + FreeSize uint64 + UUID string + Tags []string +} + +// PV is Physica lVolume +type PV struct { + Name string + VgName string + Size uint64 + UUID string +} + +// LVAttributes is attributes +type LVAttributes struct { + Type VolumeType + Permissions VolumePermissions + Allocation VolumeAllocation + FixedMinor VolumeFixedMinor + State VolumeState + Open VolumeOpen + TargetType VolumeTargetType + Zeroing VolumeZeroing + Health VolumeHealth + ActivationSkipped VolumeActivationSkipped +} + +// ParseLV ... +func ParseLV(line string) (*LV, error) { + // lvs --units=b --separator="<:SEP:>" --nosuffix --noheadings -o lv_name,lv_size,lv_uuid,lv_attr,copy_percent,lv_kernel_major,lv_kernel_minor,lv_tags --nameprefixes -a + // todo: devices, lv_ancestors, lv_descendants, lv_major, lv_minor, mirror_log, modules, move_pv, origin, region_size + // seg_count, seg_size, seg_start, seg_tags, segtype, snap_percent, stripes, stripe_size + fields, err := parse(line, 8) + if err != nil { + return nil, err + } + + size, err := strconv.ParseUint(fields["LVM2_LV_SIZE"], 10, 64) + if err != nil { + return nil, err + } + + kernelMajNumber, err := strconv.ParseUint(fields["LVM2_LV_KERNEL_MAJOR"], 10, 32) + if err != nil { + return nil, err + } + + kernelMinNumber, err := strconv.ParseUint(fields["LVM2_LV_KERNEL_MINOR"], 10, 32) + if err != nil { + return nil, err + } + + attrs, err := parseAttrs(fields["LVM2_LV_ATTR"]) + if err != nil { + return nil, err + } + + return &LV{ + Name: fields["LVM2_LV_NAME"], + Size: size, + UUID: fields["LVM2_LV_UUID"], + Attributes: *attrs, + CopyPercent: fields["LVM2_COPY_PERCENT"], + ActualDevMajNumber: uint32(kernelMajNumber), + ActualDevMinNumber: uint32(kernelMinNumber), + Tags: strings.Split(fields["LVM2_LV_TAGS"], ","), + }, nil + +} + +// ParseVG parse volume group +func ParseVG(line string) (*VG, error) { + // vgs --units=b --separator="<:SEP:>" --nosuffix --noheadings -o vg_name,vg_size,vg_free,vg_uuid,vg_tags --nameprefixes -a + fields, err := parse(line, 5) + if err != nil { + return nil, err + } + + size, err := strconv.ParseUint(fields["LVM2_VG_SIZE"], 10, 64) + if err != nil { + return nil, err + } + + freeSize, err := strconv.ParseUint(fields["LVM2_VG_FREE"], 10, 64) + if err != nil { + return nil, err + } + + return &VG{ + Name: fields["LVM2_VG_NAME"], + Size: size, + FreeSize: freeSize, + UUID: fields["LVM2_VG_UUID"], + Tags: strings.Split(fields["LVM2_VG_TAGS"], ","), + }, nil +} + +// ParsePV parse volume group +func ParsePV(line string) (*PV, error) { + // vgs --units=b --separator="<:SEP:>" --nosuffix --noheadings -o vg_name,vg_size,vg_free,vg_uuid,vg_tags --nameprefixes -a + fields, err := parse(line, 4) + if err != nil { + return nil, err + } + + size, err := strconv.ParseUint(fields["LVM2_PV_SIZE"], 10, 64) + if err != nil { + return nil, err + } + + return &PV{ + Name: fields["LVM2_PV_NAME"], + VgName: fields["LVM2_VG_NAME"], + Size: size, + UUID: fields["LVM2_PV_UUID"], + }, nil +} + +func parse(line string, numComponents int) (map[string]string, error) { + components := strings.Split(line, separator) + if len(components) != numComponents { + return nil, fmt.Errorf("expected %d components, got %d", numComponents, len(components)) + } + + fields := map[string]string{} + for _, c := range components { + idx := strings.Index(c, "=") + if idx == -1 { + return nil, fmt.Errorf("failed to parse component '%s'", c) + } + key := c[0:idx] + value := c[idx+1:] + if len(value) < 2 { + return nil, fmt.Errorf("failed to parse component '%s'", c) + } + if value[0] != '\'' || value[len(value)-1] != '\'' { + return nil, fmt.Errorf("failed to parse component '%s'", c) + } + value = value[1 : len(value)-1] + fields[key] = value + } + + return fields, nil +} + +func parseAttrs(attrs string) (*LVAttributes, error) { + if len(attrs) != 10 { + return nil, fmt.Errorf("incorrect attrs block size, expected 10, got %d in %s", len(attrs), attrs) + } + + ret := &LVAttributes{} + ret.Type = VolumeType(attrs[0]) + ret.Permissions = VolumePermissions(attrs[1]) + ret.Allocation = VolumeAllocation(attrs[2]) + ret.FixedMinor = VolumeFixedMinor(attrs[3]) + ret.State = VolumeState(attrs[4]) + ret.Open = VolumeOpen(attrs[5]) + ret.TargetType = VolumeTargetType(attrs[6]) + ret.Zeroing = VolumeZeroing(attrs[7]) + ret.Health = VolumeHealth(attrs[8]) + ret.ActivationSkipped = VolumeActivationSkipped(attrs[9]) + + return ret, nil +} diff --git a/pkg/signals/signal_posix.go b/pkg/signals/signal_posix.go new file mode 100644 index 0000000..1bafca8 --- /dev/null +++ b/pkg/signals/signal_posix.go @@ -0,0 +1,25 @@ +// +build !windows + +/* +Copyright 2021 The OpenYurt Authors. +Copyright 2018 The Knative Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package signals + +import ( + "os" + "syscall" +) + +var shutdownSignals = []os.Signal{os.Interrupt, syscall.SIGTERM} diff --git a/pkg/signals/signals.go b/pkg/signals/signals.go new file mode 100644 index 0000000..3e52837 --- /dev/null +++ b/pkg/signals/signals.go @@ -0,0 +1,82 @@ +/* +Copyright 2021 The OpenYurt Authors. +Copyright 2018 The Knative Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package signals + +import ( + "context" + "errors" + "os" + "os/signal" + "time" +) + +var onlyOneSignalHandler = make(chan struct{}) + +// SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned +// which is closed on one of these signals. If a second signal is caught, the program +// is terminated with exit code 1. +func SetupSignalHandler() (stopCh <-chan struct{}) { + close(onlyOneSignalHandler) // panics when called twice + + stop := make(chan struct{}) + c := make(chan os.Signal, 2) + signal.Notify(c, shutdownSignals...) + go func() { + <-c + close(stop) + <-c + os.Exit(1) // second signal. Exit directly. + }() + + return stop +} + +// NewContext creates a new context with SetupSignalHandler() +// as our Done() channel. +func NewContext() context.Context { + return &signalContext{stopCh: SetupSignalHandler()} +} + +type signalContext struct { + stopCh <-chan struct{} +} + +// Deadline implements context.Context +func (scc *signalContext) Deadline() (deadline time.Time, ok bool) { + return +} + +// Done implements context.Context +func (scc *signalContext) Done() <-chan struct{} { + return scc.stopCh +} + +// Err implements context.Context +func (scc *signalContext) Err() error { + select { + case _, ok := <-scc.Done(): + if !ok { + return errors.New("received a termination signal") + } + default: + } + return nil +} + +// Value implements context.Context +func (scc *signalContext) Value(key interface{}) interface{} { + return nil +} diff --git a/pkg/utils/lvm.go b/pkg/utils/lvm.go new file mode 100644 index 0000000..72aa83c --- /dev/null +++ b/pkg/utils/lvm.go @@ -0,0 +1,297 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "errors" + "fmt" + "strings" + + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +const ( + ProtectedTagName = "protected" +) + +// LVM ... +type LVM interface { + ListLV(listSpec string) ([]*model.LV, error) + CreateLV(vg, name string, size uint64, mirrors uint32, tags []string) (string, error) + RemoveLV(vg, name string) (string, error) + CloneLV(src, dest string) (string, error) + ListVG() ([]*model.VG, error) + ListPhysicalVolume() ([]*model.PV, error) + CreateVG(name, physicalVolume string, tags []string) (string, error) + ExtendVG(name, physicalVolume string) (string, error) + RemoveVG(name string) (string, error) + AddTagLV(vg, name string, tags []string) (string, error) + RemoveTagLV(vg, name string, tags []string) (string, error) +} + +// NodeLVM ... +type NodeLVM struct { +} + +// NewNodeLVM ... +func NewNodeLVM() *NodeLVM { + return &NodeLVM{} +} + +// ListLV ... +func (nl *NodeLVM) ListLV(listSpec string) ([]*model.LV, error) { + cmdList := []string{NsenterCmd, "lvs", "--units=b", "--separator=\"<:SEP:>\"", "--nosuffix", "--noheadings", + "-o", "lv_name,lv_size,lv_uuid,lv_attr,copy_percent,lv_kernel_major,lv_kernel_minor,lv_tags", "--nameprefixes", "-a", listSpec} + cmd := strings.Join(cmdList, " ") + out, err := Run(cmd) + if err != nil { + return nil, err + } + outStr := strings.TrimSpace(string(out)) + if outStr == "" { + lvs := make([]*model.LV, 0) + return lvs, nil + } + outLines := strings.Split(outStr, "\n") + lvs := []*model.LV{} + for _, line := range outLines { + line = strings.TrimSpace(line) + if !strings.Contains(line, "LVM2_LV_NAME") { + continue + } + lv, err := model.ParseLV(line) + if err != nil { + return nil, errors.New("Parse LVM: " + line + ", with error: " + err.Error()) + } + lvs = append(lvs, lv) + } + return lvs, nil +} + +// CreateLV ... +func (nl *NodeLVM) CreateLV(vg, name string, size uint64, mirrors uint32, tags []string) (string, error) { + if size == 0 { + return "", errors.New("size must be greater than 0") + } + + args := []string{"lvcreate", "-v", "-n", name, "-L", fmt.Sprintf("%db", size)} + if mirrors > 0 { + args = append(args, "-m", fmt.Sprintf("%d", mirrors), "--nosync") + } + for _, tag := range tags { + args = append(args, "--add-tag", tag) + } + + args = append(args, vg) + cmd := strings.Join(args, " ") + out, err := Run(cmd) + return string(out), err +} + +// RemoveLV ... +func (nl *NodeLVM) RemoveLV(vg, name string) (string, error) { + lvs, err := nl.ListLV(fmt.Sprintf("%s/%s", vg, name)) + if err != nil { + return "", fmt.Errorf("failed to list LVs: %v", err) + } + if len(lvs) == 0 { + return "lvm " + vg + "/" + name + " is not exist, skip remove", nil + } + if len(lvs) != 1 { + return "", fmt.Errorf("expected 1 LV, got %d", len(lvs)) + } + for _, tag := range lvs[0].Tags { + if tag == ProtectedTagName { + return "", errors.New("volume is protected") + } + } + + args := []string{NsenterCmd, "lvremove", "-v", "-f", fmt.Sprintf("%s/%s", vg, name)} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err + +} + +// CloneLV ... +func (nl *NodeLVM) CloneLV(src, dest string) (string, error) { + args := []string{NsenterCmd, "dd", fmt.Sprintf("if=%s", src), fmt.Sprintf("of=%s", dest), "bs=4M"} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err +} + +// ListVG ... +func (nl *NodeLVM) ListVG() ([]*model.VG, error) { + args := []string{NsenterCmd, "vgs", "--units=b", "--separator=\"<:SEP:>\"", "--nosuffix", "--noheadings", + "-o", "vg_name,vg_size,vg_free,vg_uuid,vg_tags", "--nameprefixes", "-a"} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + if err != nil { + return nil, err + } + vgs := []*model.VG{} + outStr := strings.TrimSpace(string(out)) + if outStr == "" { + return vgs, nil + } + outLines := strings.Split(outStr, "\n") + for _, line := range outLines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "WARNING") { + continue + } + vg, err := model.ParseVG(line) + if err != nil { + return nil, err + } + vgs = append(vgs, vg) + } + return vgs, nil +} + +// ListPhysicalVolume ... +func (nl *NodeLVM) ListPhysicalVolume() ([]*model.PV, error) { + args := []string{NsenterCmd, "pvs", "--units=b", "--separator=\"<:SEP:>\"", "--nosuffix", "--noheadings", + "-o", "vg_name,pv_name,pv_size,pv_uuid", "--nameprefixes", "-a"} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + if err != nil { + return nil, err + } + outStr := strings.TrimSpace(string(out)) + pvs := []*model.PV{} + if outStr == "" { + return pvs, nil + } + outLines := strings.Split(outStr, "\n") + for _, line := range outLines { + line = strings.TrimSpace(line) + if strings.HasPrefix(line, "WARNING") { + continue + } + pv, err := model.ParsePV(line) + if err != nil { + return nil, err + } + if pv.VgName != "" && pv.Name != "" { + pvs = append(pvs, pv) + } + } + return pvs, nil +} + +// CreateVG ... +func (nl *NodeLVM) CreateVG(name, physicalVolume string, tags []string) (string, error) { + args := []string{NsenterCmd, "vgcreate", name, physicalVolume, "-v"} + for _, tag := range tags { + args = append(args, "--add-tag", tag) + } + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err +} + +// ExtendVG ... +func (nl *NodeLVM) ExtendVG(name, physicalVolume string) (string, error) { + args := []string{NsenterCmd, "vgextend", name, physicalVolume, "-v"} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err +} + +// RemoveVG ... +func (nl *NodeLVM) RemoveVG(name string) (string, error) { + vgs, err := nl.ListVG() + if err != nil { + return "", fmt.Errorf("failed to list VGs: %v", err) + } + var vg *model.VG + for _, v := range vgs { + if v.Name == name { + vg = v + break + } + } + if vg == nil { + return "", fmt.Errorf("could not find vg to delete") + } + for _, tag := range vg.Tags { + if tag == ProtectedTagName { + return "", errors.New("volume is protected") + } + } + + args := []string{NsenterCmd, "vgremove", "-v", "-f", name} + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err + +} + +// AddTagLV ... +func (nl *NodeLVM) AddTagLV(vg, name string, tags []string) (string, error) { + lvs, err := nl.ListLV(fmt.Sprintf("%s/%s", vg, name)) + if err != nil { + return "", fmt.Errorf("failed to list LVs: %v", err) + } + if len(lvs) != 1 { + return "", fmt.Errorf("expected 1 LV, got %d", len(lvs)) + } + + args := make([]string, 0) + args = append(args, NsenterCmd) + args = append(args, "lvchange") + for _, tag := range tags { + args = append(args, "--addtag", tag) + } + + args = append(args, fmt.Sprintf("%s/%s", vg, name)) + cmd := strings.Join(args, " ") + out, err := Run(cmd) + + return string(out), err +} + +// RemoveTagLV .... +func (nl *NodeLVM) RemoveTagLV(vg, name string, tags []string) (string, error) { + + lvs, err := nl.ListLV(fmt.Sprintf("%s/%s", vg, name)) + if err != nil { + return "", fmt.Errorf("failed to list LVs: %v", err) + } + if len(lvs) != 1 { + return "", fmt.Errorf("expected 1 LV, got %d", len(lvs)) + } + + args := make([]string, 0) + args = append(args, NsenterCmd) + args = append(args, "lvchange") + for _, tag := range tags { + args = append(args, "--deltag", tag) + } + + args = append(args, fmt.Sprintf("%s/%s", vg, name)) + cmd := strings.Join(args, " ") + out, err := Run(cmd) + return string(out), err +} diff --git a/pkg/utils/lvm_mock.go b/pkg/utils/lvm_mock.go new file mode 100644 index 0000000..03b1aba --- /dev/null +++ b/pkg/utils/lvm_mock.go @@ -0,0 +1,214 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "reflect" + + "github.com/golang/mock/gomock" + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +// MockLVM ... +type MockLVM struct { + ctrl *gomock.Controller + recorder *MockLVMMockRecorder +} + +// MockLVMMockRecorder ... +type MockLVMMockRecorder struct { + mock *MockLVM +} + +// EXPECT ... +func (m *MockLVM) EXPECT() *MockLVMMockRecorder { + return m.recorder +} + +// NewMockLVM ... +func NewMockLVM(ctrl *gomock.Controller) *MockLVM { + mock := &MockLVM{ctrl: ctrl} + mock.recorder = &MockLVMMockRecorder{mock} + return mock +} + +// ListLV ... +func (m *MockLVM) ListLV(listSpec string) ([]*model.LV, error) { + + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListLV", listSpec) + ret0, _ := ret[0].([]*model.LV) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListLV ... +func (mr *MockLVMMockRecorder) ListLV(arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListLV", reflect.TypeOf((*MockLVM)(nil).ListLV), arg1) +} + +// CreateLV ... +func (m *MockLVM) CreateLV(vg, name string, size uint64, mirrors uint32, tags []string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateLV", vg, name, size, mirrors, tags) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateLV ... +func (mr *MockLVMMockRecorder) CreateLV(arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateLV", reflect.TypeOf((*MockLVM)(nil).CreateLV), arg1, arg2, arg3, arg4, arg5) +} + +// RemoveLV ... +func (m *MockLVM) RemoveLV(vg, name string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveLV", vg, name) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveLV ... +func (mr *MockLVMMockRecorder) RemoveLV(arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveLV", reflect.TypeOf((*MockLVM)(nil).RemoveLV), arg1, arg2) +} + +// CloneLV ... +func (m *MockLVM) CloneLV(src, dest string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CloneLV", src, dest) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CloneLV ... +func (mr *MockLVMMockRecorder) CloneLV(arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CloneLV", reflect.TypeOf((*MockLVM)(nil).CloneLV), arg1, arg2) +} + +// ListVG ... +func (m *MockLVM) ListVG() ([]*model.VG, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListVG") + ret0, _ := ret[0].([]*model.VG) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListVG ... +func (mr *MockLVMMockRecorder) ListVG() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListVG", reflect.TypeOf((*MockLVM)(nil).ListVG)) +} + +// ListPhysicalVolume ... +func (m *MockLVM) ListPhysicalVolume() ([]*model.PV, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListPhysicalVolume") + ret0, _ := ret[0].([]*model.PV) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListPhysicalVolume ... +func (mr *MockLVMMockRecorder) ListPhysicalVolume() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListPhysicalVolume", reflect.TypeOf((*MockLVM)(nil).ListPhysicalVolume)) +} + +// CreateVG ... +func (m *MockLVM) CreateVG(name, physicalVolume string, tags []string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateVG", name, physicalVolume, tags) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 + +} + +// CreateVG ... +func (mr *MockLVMMockRecorder) CreateVG(arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateVG", reflect.TypeOf((*MockLVM)(nil).CreateVG), arg1, arg2, arg3) +} + +// ExtendVG ... +func (m *MockLVM) ExtendVG(name, physicalVolume string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExtendVG", name, physicalVolume) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ExtendVG ... +func (mr *MockLVMMockRecorder) ExtendVG(arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExtendVG", reflect.TypeOf((*MockLVM)(nil).ExtendVG), arg1, arg2) +} + +// RemoveVG ... +func (m *MockLVM) RemoveVG(name string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExtendVG", name) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveVG ... +func (mr *MockLVMMockRecorder) RemoveVG(arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveVG", reflect.TypeOf((*MockLVM)(nil).RemoveVG), arg1) +} + +// AddTagLV ... +func (m *MockLVM) AddTagLV(vg, name string, tags []string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddTagLV", vg, name, tags) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddTagLV ... +func (mr *MockLVMMockRecorder) AddTagLV(arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddTagLV", reflect.TypeOf((*MockLVM)(nil).AddTagLV), arg1, arg2, arg3) +} + +// RemoveTagLV ... +func (m *MockLVM) RemoveTagLV(vg, name string, tags []string) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveTagLV", vg, name, tags) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RemoveTagLV ... +func (mr *MockLVMMockRecorder) RemoveTagLV(arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveTagLV", reflect.TypeOf((*MockLVM)(nil).RemoveTagLV), arg1, arg2, arg3) +} diff --git a/pkg/utils/mounter.go b/pkg/utils/mounter.go new file mode 100644 index 0000000..56f2110 --- /dev/null +++ b/pkg/utils/mounter.go @@ -0,0 +1,263 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "errors" + "fmt" + "io" + "os" + "os/exec" + "strings" + + log "github.com/sirupsen/logrus" + utilexec "k8s.io/utils/exec" + k8smount "k8s.io/utils/mount" +) + +const ( + // fsckErrorsCorrected tag + fsckErrorsCorrected = 1 + // fsckErrorsUncorrected tag + fsckErrorsUncorrected = 4 +) + +type findmntResponse struct { + FileSystems []fileSystem `json:"filesystems"` +} + +type fileSystem struct { + Target string `json:"target"` + Propagation string `json:"propagation"` + FsType string `json:"fstype"` + Options string `json:"options"` +} + +// Mounter is responsible for formatting and mounting volumes +type Mounter interface { + k8smount.Interface + utilexec.Interface + // If the folder doesn't exist, it will call 'mkdir -p' + EnsureFolder(string) error + // FormatAndMount ... + FormatAndMount(string, string, string, []string, string) error + + // IsMounted checks whether the target path is a correct mount (i.e: + // propagated). It returns true if it's mounted. An error is returned in + // case of system errors or if it's mounted incorrectly. + IsMounted(target string) (bool, error) + + SafePathRemove(target string) error +} + +// TODO(arslan): this is Linux only for now. Refactor this into a package with +// architecture specific code in the future, such as mounter_darwin.go, +// mounter_linux.go, etc.. +type NodeMounter struct { + k8smount.SafeFormatAndMount + utilexec.Interface +} + +// NewMounter returns a new mounter instance +func NewMounter() Mounter { + return &NodeMounter{ + k8smount.SafeFormatAndMount{ + Interface: k8smount.New(""), + Exec: utilexec.New(), + }, + utilexec.New(), + } +} + +// EnsureFolder ... +func (m *NodeMounter) EnsureFolder(target string) error { + mkdirCmd := "mkdir" + _, err := m.LookPath(mkdirCmd) + if err != nil { + if err == exec.ErrNotFound { + return fmt.Errorf("%q executable not found in $PATH", mkdirCmd) + } + return err + } + + mkdirCmd = NsenterCmd + mkdirCmd + mkdirCmd += fmt.Sprintf(" -p %s", target) + log.Infof("mkdir for folder, the command is %s", mkdirCmd) + output, err := Run(mkdirCmd) + if err != nil { + return fmt.Errorf("EnsureFolder:: mkdir for folder output: %s error: %v", output, err) + } + return nil +} + +// FormatAndMount ... +func (m *NodeMounter) FormatAndMount(source, target, fstype string, mkfsOptions []string, mountOptions string) error { + // diskMounter.Interface = m.K8smounter + readOnly := false + + if !readOnly { + // Run fsck on the disk to fix repairable issues, only do this for volumes requested as rw. + args := []string{"-a", source} + + out, err := m.Exec.Command("fsck", args...).CombinedOutput() + if err != nil { + ee, isExitError := err.(utilexec.ExitError) + switch { + case err == utilexec.ErrExecutableNotFound: + log.Warningf("'fsck' not found on system; continuing mount without running 'fsck'.") + case isExitError && ee.ExitStatus() == fsckErrorsCorrected: + log.Infof("Device %s has errors which were corrected by fsck.", source) + case isExitError && ee.ExitStatus() == fsckErrorsUncorrected: + return fmt.Errorf("'fsck' found errors on device %s but could not correct them: %s", source, string(out)) + case isExitError && ee.ExitStatus() > fsckErrorsUncorrected: + } + } + } + + // Try to mount the disk + cmd := fmt.Sprintf("%smount -o %s %s %s", NsenterCmd, mountOptions, source, target) + log.Infof("FormatAndMount:: cmd: %s", cmd) + output, mountErr := Run(cmd) + if mountErr != nil { + // Mount failed. This indicates either that the disk is unformatted or + // it contains an unexpected filesystem. + existingFormat, err := m.GetDiskFormat(source) + + if err != nil { + return err + } + if existingFormat == "" { + if readOnly { + // Don't attempt to format if mounting as readonly, return an error to reflect this. + return errors.New("failed to mount unformatted volume as read only") + } + + // Disk is unformatted so format it. + args := []string{source} + // Use 'ext4' as the default + if len(fstype) == 0 { + fstype = "ext4" + } + + if fstype == "ext4" || fstype == "ext3" { + args = []string{ + "-F", // Force flag + "-m0", // Zero blocks reserved for super-user + source, + } + // add mkfs options + if len(mkfsOptions) != 0 { + args = []string{} + for _, opts := range mkfsOptions { + args = append(args, opts) + } + args = append(args, source) + } + } + log.Infof("Disk %q appears to be unformatted, attempting to format as type: %q with options: %v", source, fstype, args) + + mkfsCmd := fmt.Sprintf("%s mkfs.%s %s", NsenterCmd, fstype, strings.Join(args, " ")) + log.Infof("FormatAndMount:: mkfscmd: %s", mkfsCmd) + _, err = Run(mkfsCmd) + if err == nil { + // the disk has been formatted successfully try to mount it again. + output, mountErr := Run(cmd) + log.Infof("FormatAndMount:: cmd output %s", output) + return mountErr + } + log.Errorf("format of disk %q failed: type:(%q) target:(%q) options:(%q) output: (%s) error:(%v)", source, fstype, target, mkfsOptions, output, err) + return err + } + // Disk is already formatted and failed to mount + if len(fstype) == 0 || fstype == existingFormat { + // This is mount error + return mountErr + } + // Block device is formatted with unexpected filesystem, let the user know + return fmt.Errorf("failed to mount the volume as %q, it already contains %s. Mount error: %v", fstype, existingFormat, mountErr) + } + + return mountErr +} + +// IsMounted ... +func (m *NodeMounter) IsMounted(target string) (bool, error) { + if target == "" { + return false, errors.New("target is not specified for checking the mount") + } + findmntCmd := "grep" + findmntArgs := []string{target, "/proc/mounts"} + out, err := exec.Command(findmntCmd, findmntArgs...).CombinedOutput() + outStr := strings.TrimSpace(string(out)) + if err != nil { + if outStr == "" { + return false, nil + } + return false, fmt.Errorf("checking mounted failed: %v cmd: %q output: %q", + err, findmntCmd, outStr) + } + if strings.Contains(outStr, target) { + return true, nil + } + return false, nil +} + +// SafePathRemove ... +func (m *NodeMounter) SafePathRemove(targetPath string) error { + fo, err := os.Lstat(targetPath) + if err != nil { + return err + } + isMounted, err := m.IsMounted(targetPath) + if err != nil { + return err + } + if isMounted { + return errors.New("Path is mounted, not remove: " + targetPath) + } + if fo.IsDir() { + empty, err := IsDirEmpty(targetPath) + if err != nil { + return errors.New("Check path empty error: " + targetPath + err.Error()) + } + if !empty { + return errors.New("Cannot remove Path not empty: " + targetPath) + } + } + err = os.Remove(targetPath) + if err != nil { + return err + } + return nil +} + +// IsDirEmpty return status of dir empty or not +func IsDirEmpty(name string) (bool, error) { + f, err := os.Open(name) + if err != nil { + return false, err + } + defer f.Close() + + // read in ONLY one file + _, err = f.Readdir(1) + // and if the file is EOF... well, the dir is empty. + if err == io.EOF { + return true, nil + } + return false, err +} diff --git a/pkg/utils/mounter_mock.go b/pkg/utils/mounter_mock.go new file mode 100644 index 0000000..9f67cf6 --- /dev/null +++ b/pkg/utils/mounter_mock.go @@ -0,0 +1,107 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "reflect" + + "github.com/golang/mock/gomock" + utilexec "k8s.io/utils/exec" + k8smount "k8s.io/utils/mount" +) + +// MockMounter ... +type MockMounter struct { + k8smount.SafeFormatAndMount + utilexec.Interface + ctrl *gomock.Controller + recorder *MockMounterMockRecorder +} + +// MockMounterMockRecorder ... +type MockMounterMockRecorder struct { + mock *MockMounter +} + +// NewMockMounter ... +func NewMockMounter(ctrl *gomock.Controller) *MockMounter { + mock := &MockMounter{ctrl: ctrl} + mock.recorder = &MockMounterMockRecorder{mock} + return mock +} + +// EXPECT ... +func (m *MockMounter) EXPECT() *MockMounterMockRecorder { + return m.recorder +} + +// EnsureFolder ... +func (m *MockMounter) EnsureFolder(target string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureFolder", target) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureFolder ... +func (mr MockMounterMockRecorder) EnsureFolder(target interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureFolder", reflect.TypeOf((*MockMounter)(nil).EnsureFolder), target) +} + +// FormatAndMount ... +func (m *MockMounter) FormatAndMount(source, target, fstype string, mkfsOptions []string, mountOptions string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "FormatAndMount", source, target, fstype, mkfsOptions, mountOptions) + ret0, _ := ret[0].(error) + return ret0 +} + +// FormatAndMount ... +func (mr MockMounterMockRecorder) FormatAndMount(arg1, arg2, arg3, arg4, arg5 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "FormatAndMount", reflect.TypeOf((*MockMounter)(nil).FormatAndMount), arg1, arg2, arg3, arg4, arg5) +} + +// IsMounted ... +func (m *MockMounter) IsMounted(target string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "IsMounted", target) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// IsMounted ... +func (mr MockMounterMockRecorder) IsMounted(target interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsMounted", reflect.TypeOf((*MockMounter)(nil).IsMounted), target) +} + +// SafePathRemove ... +func (m MockMounter) SafePathRemove(targetPath string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SafePathRemove", targetPath) + ret0, _ := ret[0].(error) + return ret0 +} + +// SafePathRemove ... +func (mr MockMounterMockRecorder) SafePathRemove(target interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SafePathRemove", reflect.TypeOf((*MockMounter)(nil).SafePathRemove), target) +} diff --git a/pkg/utils/pmem.go b/pkg/utils/pmem.go new file mode 100644 index 0000000..af7d6b0 --- /dev/null +++ b/pkg/utils/pmem.go @@ -0,0 +1,173 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "encoding/json" + "errors" + "fmt" + "strings" + + "github.com/openyurtio/node-resource-manager/pkg/model" + log "github.com/sirupsen/logrus" +) + +// Pmemer ... +type Pmemer interface { + GetRegions() (*model.PmemRegions, error) + CreateNamespace(string, string) error + CheckNamespaceUsed(string) bool + GetPmemNamespaceDeivcePath(string, string) (string, string, error) + MakeNamespaceMemory(chardev string) error + CheckKMEMCreated(chardev string) (bool, error) +} + +// NodePmemer ... +type NodePmemer struct { +} + +// NewNodePmemer create new NodePmemer struct +func NewNodePmemer() *NodePmemer { + return &NodePmemer{} +} + +// GetRegions ... +func (np *NodePmemer) GetRegions() (*model.PmemRegions, error) { + regions := &model.PmemRegions{} + getRegionCmd := fmt.Sprintf("%s ndctl list -RN", NsenterCmd) + regionOut, err := Run(getRegionCmd) + if err != nil { + return regions, err + } + err = json.Unmarshal(([]byte)(regionOut), regions) + if err != nil { + if strings.HasPrefix(regionOut, "[") { + regionList := []model.PmemRegion{} + err = json.Unmarshal(([]byte)(regionOut), ®ionList) + if err != nil { + return regions, err + } + regions.Regions = regionList + } else { + return regions, err + } + } + return regions, nil +} + +// CreateNamespace ... +func (np *NodePmemer) CreateNamespace(region, pmemType string) error { + var createCmd string + if pmemType == "lvm" { + createCmd = fmt.Sprintf("%s ndctl create-namespace -r %s", NsenterCmd, region) + } else { + createCmd = fmt.Sprintf("%s ndctl create-namespace -r %s --mode=devdax", NsenterCmd, region) + } + _, err := Run(createCmd) + if err != nil { + log.Errorf("Create NameSpace for region %s error: %v", region, err) + return err + } + log.Infof("Create NameSpace for region %s successful", region) + return nil +} + +// CheckNamespaceUsed device used in block +func (np *NodePmemer) CheckNamespaceUsed(devicePath string) bool { + pvCheckCmd := fmt.Sprintf("%s pvs %s 2>&1 | grep -v \"Failed to \" | grep /dev | awk '{print $2}' | wc -l", NsenterCmd, devicePath) + out, err := Run(pvCheckCmd) + if err == nil && strings.TrimSpace(out) != "0" { + log.Infof("CheckNamespaceUsed: NameSpace %s used for pv", devicePath) + return true + } + + out, err = checkFSType(devicePath) + if err == nil && strings.TrimSpace(out) != "" { + log.Infof("CheckNamespaceUsed: NameSpace %s format as %s", devicePath, out) + return true + } + return false +} + +// GetPmemNamespaceDeivcePath ... +func (np *NodePmemer) GetPmemNamespaceDeivcePath(region, mode string) (devicePath string, namespaceName string, err error) { + regions, err := np.getRegionNamespaceInfo(region) + if err != nil { + return "", "", err + } + namespace := regions.Regions[0].Namespaces[0] + if namespace.Mode != mode { + log.Errorf("GetPmemNamespaceDeivcePath namespace mode %s wrong with: %s", namespace.Mode, mode) + return "", "", errors.New("GetPmemNamespaceDeivcePath pmem namespace wrong mode" + namespace.Mode) + } + if mode == "fsdax" { + devicePath = "/dev/" + namespace.BlockDev + } else { + devicePath = "/dev/" + namespace.CharDev + } + return devicePath, namespace.Dev, nil +} + +func (np *NodePmemer) getRegionNamespaceInfo(region string) (*model.PmemRegions, error) { + listCmd := fmt.Sprintf("%s ndctl list -RN -r %s", NsenterCmd, region) + + out, err := Run(listCmd) + if err != nil { + log.Errorf("List NameSpace for region %s error: %v", region, err) + return nil, err + } + regions := &model.PmemRegions{} + err = json.Unmarshal(([]byte)(out), regions) + if len(regions.Regions) == 0 { + log.Errorf("list Namespace for region %s get 0 region, out: %s", region, out) + return nil, errors.New("list Namespace get 0 region by " + region) + } + + if len(regions.Regions[0].Namespaces) != 1 { + log.Errorf("list Namespace for region %s get 0 or multi namespaces", region) + return nil, errors.New("list Namespace for region get 0 or multi namespaces" + region) + } + return regions, nil +} + +// MakeNamespaceMemory ... +func (np *NodePmemer) MakeNamespaceMemory(chardev string) error { + makeCmd := fmt.Sprintf("%s daxctl reconfigure-device -m system-ram %s", NsenterCmd, chardev) + _, err := Run(makeCmd) + return err +} + +// CheckKMEMCreated ... +func (np *NodePmemer) CheckKMEMCreated(chardev string) (bool, error) { + listCmd := fmt.Sprintf("%s daxctl list", NsenterCmd) + out, err := Run(listCmd) + if err != nil { + log.Errorf("CheckKMEMCreated:: List daxctl error: %v", err) + return false, err + } + memList := []*model.DaxctrlMem{} + err = json.Unmarshal(([]byte)(out), &memList) + if err != nil { + return false, err + } + for _, mem := range memList { + if mem.Chardev == chardev && mem.Mode == "system-ram" { + return true, nil + } + } + return false, nil +} diff --git a/pkg/utils/pmem_mock.go b/pkg/utils/pmem_mock.go new file mode 100644 index 0000000..481c15a --- /dev/null +++ b/pkg/utils/pmem_mock.go @@ -0,0 +1,135 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "reflect" + + "github.com/golang/mock/gomock" + "github.com/openyurtio/node-resource-manager/pkg/model" +) + +// MockPmemer .. +type MockPmemer struct { + ctrl *gomock.Controller + recorder *MockPmemerMockRecorder +} + +// MockPmemerMockRecorder ... +type MockPmemerMockRecorder struct { + mock *MockPmemer +} + +// EXPECT ... +func (m *MockPmemer) EXPECT() *MockPmemerMockRecorder { + return m.recorder +} + +// NewMockPmemer ... +func NewMockPmemer(ctrl *gomock.Controller) *MockPmemer { + mock := &MockPmemer{ctrl: ctrl} + mock.recorder = &MockPmemerMockRecorder{mock} + return mock +} + +// GetRegions ... +func (m *MockPmemer) GetRegions() (*model.PmemRegions, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRegions") + ret0, _ := ret[0].(*model.PmemRegions) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetRegions ... +func (mr *MockPmemerMockRecorder) GetRegions() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRegions", reflect.TypeOf((*MockPmemer)(nil).GetRegions)) +} + +// CreateNamespace ... +func (m *MockPmemer) CreateNamespace(arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateNamespace", arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateNamespace ... +func (mr *MockPmemerMockRecorder) CreateNamespace(arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateNamespace", reflect.TypeOf((*MockPmemer)(nil).CreateNamespace), arg1, arg2) +} + +// CheckNamespaceUsed ... +func (m *MockPmemer) CheckNamespaceUsed(arg1 string) bool { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckNamespaceUsed", arg1) + ret0, _ := ret[0].(bool) + return ret0 +} + +// CheckNamespaceUsed ... +func (mr *MockPmemerMockRecorder) CheckNamespaceUsed(arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckNamespaceUsed", reflect.TypeOf((*MockPmemer)(nil).CheckNamespaceUsed), arg1) +} + +// GetPmemNamespaceDeivcePath ... +func (m *MockPmemer) GetPmemNamespaceDeivcePath(arg1, arg2 string) (string, string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetPmemNamespaceDeivcePath", arg1, arg2) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 +} + +// GetPmemNamespaceDeivcePath ... +func (mr *MockPmemerMockRecorder) GetPmemNamespaceDeivcePath(arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetPmemNamespaceDeivcePath", reflect.TypeOf((*MockPmemer)(nil).GetPmemNamespaceDeivcePath), arg1, arg2) +} + +// CheckKMEMCreated ... +func (m *MockPmemer) CheckKMEMCreated(arg1 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CheckKMEMCreated", arg1) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CheckKMEMCreated ... +func (mr *MockPmemerMockRecorder) CheckKMEMCreated(arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CheckKMEMCreated", reflect.TypeOf((*MockPmemer)(nil).CheckKMEMCreated), arg1) +} + +// MakeNamespaceMemory ... +func (m *MockPmemer) MakeNamespaceMemory(arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MakeNamespaceMemory", arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// MakeNamespaceMemory ... +func (mr *MockPmemerMockRecorder) MakeNamespaceMemory(arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MakeNamespaceMemory", reflect.TypeOf((*MockPmemer)(nil).MakeNamespaceMemory), arg1) +} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go new file mode 100644 index 0000000..ca59b50 --- /dev/null +++ b/pkg/utils/utils.go @@ -0,0 +1,176 @@ +/* +Copyright 2021 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "errors" + "fmt" + "io/ioutil" + "net/http" + "os/exec" + "path/filepath" + "strings" + + "github.com/openyurtio/node-resource-manager/pkg/model" + log "github.com/sirupsen/logrus" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +const ( + // MetadataURL is metadata url + MetadataURL = "http://100.100.100.200/latest/meta-data/" + + // NsenterCmd use to init resource + NsenterCmd = "/usr/bin/nsenter --mount=/proc/1/ns/mnt --ipc=/proc/1/ns/ipc --net=/proc/1/ns/net --uts=/proc/1/ns/uts " +) + +// ErrParse ... +var ErrParse = errors.New("Cannot parse output of blkid") + +//GetMetaData get metadata from ecs meta-server +func GetMetaData(resource string) (string, error) { + resp, err := http.Get(MetadataURL + resource) + if err != nil { + return "", err + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} + +// Run run shell command +func Run(cmd string) (string, error) { + out, err := exec.Command("sh", "-c", cmd).CombinedOutput() + if err != nil { + return "", fmt.Errorf("Failed to run cmd: " + cmd + ", with out: " + string(out) + ", with error: " + err.Error()) + } + return string(out), nil +} + +// NodeFilter go through all configmap to find current node config +func NodeFilter(configOperator metav1.LabelSelectorOperator, configKey, configValue string, nodeInfo *v1.Node) bool { + + isMatched := false + switch configOperator { + case metav1.LabelSelectorOpIn: + for key, value := range nodeInfo.Labels { + if key == configKey && value == configValue { + isMatched = true + } + } + case metav1.LabelSelectorOpNotIn: + flag := false + for key, value := range nodeInfo.Labels { + if key == configKey && value == configValue { + flag = true + } + } + if flag == false { + isMatched = true + } + case metav1.LabelSelectorOpExists: + for key := range nodeInfo.Labels { + if key == configKey { + isMatched = true + } + } + case metav1.LabelSelectorOpDoesNotExist: + flag := false + for key := range nodeInfo.Labels { + if key == configKey { + flag = true + } + } + if flag == false { + isMatched = true + } + default: + log.Errorf("Get unsupported operator: %s", configOperator) + } + return isMatched +} + +// ConvertRegion2Namespace ... +func ConvertRegion2Namespace(region string) string { + regionIndex := region[6:] + return fmt.Sprintf("namespace%s.0", regionIndex) +} + +// ConvertNamespace2LVMDevicePath ... +func ConvertNamespace2LVMDevicePath(namespace string, regions *model.PmemRegions) string { + for _, region := range regions.Regions { + for _, actualNamespace := range region.Namespaces { + if actualNamespace.Dev == namespace { + return filepath.Join("/dev", actualNamespace.BlockDev) + } + } + } + return "" +} + +func checkFSType(devicePath string) (string, error) { + // We use `file -bsL` to determine whether any filesystem type is detected. + // If a filesystem is detected (ie., the output is not "data", we use + // `blkid` to determine what the filesystem is. We use `blkid` as `file` + // has inconvenient output. + // We do *not* use `lsblk` as that requires udev to be up-to-date which + // is often not the case when a device is erased using `dd`. + output, err := exec.Command("file", "-bsL", devicePath).CombinedOutput() + if err != nil { + return "", err + } + if strings.TrimSpace(string(output)) == "data" { + return "", nil + } + output, err = exec.Command("blkid", "-c", "/dev/null", "-o", "export", devicePath).CombinedOutput() + if err != nil { + return "", err + } + + lines := strings.Split(string(output), "\n") + for _, line := range lines { + fields := strings.Split(strings.TrimSpace(line), "=") + if len(fields) != 2 { + return "", ErrParse + } + if fields[0] == "TYPE" { + return fields[1], nil + } + } + return "", ErrParse +} + +// IsPart if smallList is part of or equal to largeList, return true; +func IsPart(largeList, smallList []string) bool { + isPartFlag := true + for _, smalltmp := range smallList { + flag := false + for _, largetmp := range largeList { + if smalltmp == largetmp { + flag = true + } + } + if flag == false { + isPartFlag = false + } + } + return isPartFlag +}