update readme and support kind to deploy kosmos

.github/workflows/dev_images.yml

@@ -0,0 +1,33 @@
+name: Dev Images
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+jobs:
+  images:
+    runs-on: ubuntu-latest
+    permissions: write-all
+    # not trigger on 'kosmos-io/kosmos'
+    if: github.repository != 'kosmos-io/kosmos'
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Set up qemu
+        uses: docker/setup-qemu-action@v2
+      - name: Login registry
+        run: |
+          echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin
+      - name: 'Get Previous tag'
+        id: previoustag
+        uses: "WyriHaximus/github-action-get-previous-tag@v1"
+        with:
+          fallback: 1.0.0
+      - name: Push images
+        env:
+          ON_PLUGINS: true
+          REGISTRY: ghcr.io/${{ github.repository_owner }}/${{ github.repository_name }}
+          TAG: ${{ steps.previoustag.outputs.tag }}
+        run: |
+          make upload-images REGISTRY=${REGISTRY%?} VERSION=$TAG

Makefile

@@ -9,6 +9,15 @@ REGISTRY_PASSWORD?=""
 REGISTRY_SERVER_ADDRESS?=""
 KIND_IMAGE_TAG?="v1.25.3"
 
+MACOS_TARGETS := clusterlink-controller-manager  \
+				kosmos-operator \
+				 clusterlink-elector \
+				clusterlink-network-manager \
+				clusterlink-proxy \
+				clustertree-cluster-manager \
+				scheduler \
+
+# clusterlink-agent and clusterlink-floater only support linux platform
 TARGETS :=  clusterlink-controller-manager  \
 			kosmos-operator \
 			clusterlink-agent \
@@ -19,6 +28,11 @@ TARGETS :=  clusterlink-controller-manager  \
 			clustertree-cluster-manager \
 			scheduler \
 
+# If GOOS is macOS, assign the value of MACOS_TARGETS to TARGETS
+ifeq ($(GOOS), darwin)
+	TARGETS := $(MACOS_TARGETS)
+endif
+
 CTL_TARGETS := kosmosctl
 
 # Build code.
@@ -82,12 +96,13 @@ multi-platform-images: $(MP_TARGET)
 
 .PHONY: clean
 clean:
-	rm -rf _tmp _output
+	hack/clean.sh
 
 .PHONY: update
 update:
 	hack/update-all.sh
 
+# verify-all.sh can not found
 .PHONY: verify
 verify:
 	hack/verify-all.sh
@@ -118,7 +133,7 @@ release:
 	@make release-kosmosctl GOOS=darwin GOARCH=arm64
 
 release-kosmosctl:
-	hack/release.sh kosmosctl ${GOOS} ${GOARCH}
+	hack/release.sh kosmosctl ${GOOS} ${GOARCH} ${VERSION}
 
 .PHONY: lint
 lint: golangci-lint

README.md

@@ -45,24 +45,105 @@ The Kosmos scheduling module is an extension developed on top of the Kubernetes
    Whether building a hybrid cloud environment or requiring flexible deployment of workloads across different clusters, the Kosmos scheduling module serves as a reliable solution, assisting users in managing containerized applications more efficiently.
 
 ## Quick Start
+
+This guide will cover:
+- Use Kind create three kubernetes cluster
+- Install `kosmosctl`, you can download from the <a href="https://github.com/kosmos-io/kosmos/releases">releases page</a> or build from source
+- Install `kosmos` control plane components in a Kubernetes cluster which is known as `host cluster`.
+- Join a member cluster to `kosmos` control plane on the `host cluster`.
+- Use the Kosmos clusters.
+
+### Prerequisites
+- [Go](https://golang.org/) version v1.20+
+- [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/) version v1.19+
+- [kind](https://kind.sigs.k8s.io/) version v0.14.0+
+
+### Deploy and run Kosmos control plane use script
+
+run the following script:
+
+```bash
+hack/local-up-kosmosctl.sh
+```
+
+### Deploy and run Kosmos step by step
+
+#### 1. Use Kind create cluster
+- Config your kind cluster use flow config, change the param as you need
+```yaml
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  # WARNING: It is _strongly_ recommended that you keep this the default
+  # (127.0.0.1) for security reasons. However it is possible to change this.
+  apiServerAddress: "192.168.200.112"
+  # By default the API server listens on a random open port.
+  # You may choose a specific port but probably don't need to in most cases.
+  # Using a random port makes it easier to spin up multiple clusters.
+  apiServerPort: 1443
+nodes:
+- role: control-plane
+  extraPortMappings:
+    - containerPort: "{{container_port}}"
+      hostPort: "{{host_port}}"
+      protocol: TCP
+      listenAddress: "{{host_ipaddress}}"
+#- role: worker
+#- role: worker
+```
+- create cluster1
+  `kind create cluster -n kind-cluster1 --config /path/to/kind-config`
+- create cluster2
+  `kind create cluster -n kind-cluster2 --config /path/to/kind-config`
+- create cluster3
+  `kind create cluster -n kind-cluster3 --config /path/to/kind-config`
+
+#### 2. Install `kosmosctl`
+
+##### 2.1 Use prebuild Binary executable file
+- Download from the <a href="https://github.com/kosmos-io/kosmos/releases">releases page</a>, only support macOS and linux
+- put `kosmosctl` to you Path, so you can execute `kosmosctl` without absolute path
+
+##### 2.2 Build from source
+- Download source
+`git clone https://github.com/kosmos-io/kosmos.git`
+- Build code, the output file is in the` <project_dir>/_output/bin/linux/amd64/kosmosctl`
+`make kosmosctl VERSION=v0.1.9`]
+- you can find any available version or tags in [here](https://github.com/kosmos-io/kosmos/tags)
+
+##### 3. Install `kosmos` control plane components
 The following command allows you to quickly run an experimental environment with three clusters. 
-Install the control plane in the host cluster.
+Install the control plane in the host cluster. 
+Please config the pod can access the kind cluster apiServer, avoid the `kosmos-operator` CrashLoopBackOff
 ```Shell
 kosmosctl install  --cni calico --default-nic eth0 (We build a network tunnel based the network interface value passed by the arg default-nic)
 ```
-
-Join the two member clusters.
+##### 4. Join to `kosmos` control plane
+- Get the cluster1 and cluster2 kubeconfig and put it on the host cluster
+- Join the two member clusters(execute on the host cluster).
 ```Shell
 kosmosctl join cluster --name cluster1 --kubeconfig ~/kubeconfig/cluster1-kubeconfig  --cni calico --default-nic eth0  --enable-all
 kosmosctl join cluster --name cluster2 --kubeconfig ~/kubeconfig/cluster2-kubeconfig  --cni calico --default-nic eth0  --enable-all
 ```
-And then we can Use the Kosmos clusters like single cluster.
+##### 5. Use the Kosmos clusters like single cluster on the control plane.
+```shell
+kubectl get nodes
+```
+- use this command we can see the two cluster nodes Name like `kosmos-*` roles as `agent`
+<div><img src="docs/images/demo-check-node.png" style="width:400px;"  alt="nodes"/></div>
 
 ## Contact
 If you have questions, feel free to reach out to us in the following ways:
 - [Email](mailto:[email protected])
 - [WeChat](./docs/images/kosmos-WeChatIMG.png)
 
+
+## Contributing
+
+If you're interested in being a contributor and want to get involved in
+developing the Karmada code, please see [CONTRIBUTING](CONTRIBUTING.md) for
+details on submitting patches and the contribution workflow.
+
 ## Contributors
 
 <a href="https://github.com/kosmos-io/clusterlink/graphs/contributors">
@@ -75,7 +156,7 @@ Made with [contrib.rocks](https://contrib.rocks).
 
 Copyright 2023 the KOSMOS Authors. All rights reserved.
 
-Licensed under the Apache License, Version 2.0.
+Licensed under the Apache License, Version 2.0. See the [LICENSE](LICENSE) file for details.
 
 
 

README_zh.md

@@ -48,7 +48,7 @@ Kosmos调度模块是基于Kubernetes调度框架的扩展开发，旨在满足
 ```bash
 kosmosctl install  --cni calico --default-nic eth0 (参数default-nic 表示基于哪个网卡创建网络隧道)
 ```
-加入两个子集群
+加入两个子集群(在kosmos控制面上执行)
 ```bash
 kosmosctl join cluster --name cluster1 --kubeconfig ~/kubeconfig/cluster1-kubeconfig  --cni calico --default-nic eth0  --enable-all
 kosmosctl join cluster --name cluster2 --kubeconfig ~/kubeconfig/cluster2-kubeconfig  --cni calico --default-nic eth0  --enable-all

deploy/clusterlink-elector.yml

@@ -65,7 +65,6 @@ spec:
             - mountPath: /etc/clusterlink
               name: proxy-config
               readOnly: true
-      serviceAccount: clusterlink-elector
       serviceAccountName: clusterlink-elector
       tolerations:
         - effect: NoSchedule

deploy/kindClusterConfig/kind-cluster1-config.yml

@@ -0,0 +1,19 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  # WARNING: It is _strongly_ recommended that you keep this the default
+  # (127.0.0.1) for security reasons. However it is possible to change this.
+  apiServerAddress: "{{host_ipaddress}}"
+  # By default the API server listens on a random open port.
+  # You may choose a specific port but probably don't need to in most cases.
+  # Using a random port makes it easier to spin up multiple clusters.
+  apiServerPort: 1443
+nodes:
+  - role: control-plane
+#- role: worker
+#- role: worker
+#  extraPortMappings:
+#    - containerPort: 6443
+#      hostPort: 1443
+#      protocol: TCP
+#      listenAddress: 192.168.200.112

deploy/kindClusterConfig/kind-cluster2-config.yml

@@ -0,0 +1,19 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  # WARNING: It is _strongly_ recommended that you keep this the default
+  # (127.0.0.1) for security reasons. However it is possible to change this.
+  apiServerAddress: "{{host_ipaddress}}"
+  # By default the API server listens on a random open port.
+  # You may choose a specific port but probably don't need to in most cases.
+  # Using a random port makes it easier to spin up multiple clusters.
+  apiServerPort: 2443
+nodes:
+  - role: control-plane
+#- role: worker
+#- role: worker
+#  extraPortMappings:
+#    - containerPort: 6443
+#      hostPort: 1443
+#      protocol: TCP
+#      listenAddress: 192.168.200.112

deploy/kindClusterConfig/kind-cluster3-config.yml

@@ -0,0 +1,19 @@
+kind: Cluster
+apiVersion: kind.x-k8s.io/v1alpha4
+networking:
+  # WARNING: It is _strongly_ recommended that you keep this the default
+  # (127.0.0.1) for security reasons. However it is possible to change this.
+  apiServerAddress: "{{host_ipaddress}}"
+  # By default the API server listens on a random open port.
+  # You may choose a specific port but probably don't need to in most cases.
+  # Using a random port makes it easier to spin up multiple clusters.
+  apiServerPort: 3443
+nodes:
+  - role: control-plane
+#- role: worker
+#- role: worker
+#  extraPortMappings:
+#    - containerPort: 6443
+#      hostPort: 1443
+#      protocol: TCP
+#      listenAddress: 192.168.200.112

deploy/scheduler/deployment.yaml

@@ -51,7 +51,6 @@ spec:
       restartPolicy: Always
       dnsPolicy: ClusterFirst
       serviceAccountName: kosmos-scheduler
-      serviceAccount: kosmos-scheduler
 
 ---
 apiVersion: v1

docs/clustertree/README.md

@@ -0,0 +1,76 @@
+# ClusterTree(集群容器编排)
+
+## 技术架构图
+<div><img src="../images/clustertree-arch.png" style="width:900px;"  alt="ClusterTree"/></div>
+
+- 所有的服务部署都需要在kosmos控制面上执行，通过kosmos的控制面将工作负载（deploy、service、PV、PVCS）
+- 所有的controller都是通过Cluster-Manager这个服务来进行管理
+- 整个ClusterTree暂时只用到Cluster这个CRD，这个CRD中主要存储整个kosmos集群中各个集群的kubeconfig配置信息
+- controller通过读取Cluster 这个CRD来读取kubeconfig然后进行调和
+
+## 详细介绍
+### 如何进行本地debug
+- 推荐使用goland进行开发和测试
+- 使用kind创建3个集群，具体参考[README中的Quick Start](../../README.md#quick-start)部分内容
+- 配置Program Arguments 为 
+`--kube-qps=500 --kube-burst=1000 --kubeconfig=$ProjectFileDir$/kube-config/kosmos-control-cluster-config.yaml --leader-elect=false --multi-cluster-service=false --v=4`
+将`--kubeconfig`改成kosmos主集群配置
+- 由于kind创建的3个集群，是通过内部的coredns进行通信，所以在Cluster-Manager进行debug或者运行的时候，需要改下config的获取，并修改IP和端口信息
+
+### Cluster-Manger
+- 整体的模块拆分如下
+- controller目录包含pod、pv、pvc的调和以及跨集群的服务导入、服务导出（实现服务的注册和服务发现）
+- extensions目录包含kosmos自身daemonset处理的controller，实现kosmos daemonset下发到不同的集群
+- node-server目录主要功能为管理整个集群的node，通过http的api提供一些服务，比如执行命令、获取日志
+
+```text
+└── cluster-manager
+    ├── cluster_controller.go
+    ├── controllers
+    │   ├── common_controller.go
+    │   ├── mcs
+    │   │   ├── auto_mcs_controller.go
+    │   │   ├── serviceexport_controller.go
+    │   │   └── serviceimport_controller.go
+    │   ├── node_lease_controller.go
+    │   ├── node_resources_controller.go
+    │   ├── pod
+    │   │   ├── leaf_pod_controller.go
+    │   │   ├── root_pod_controller.go
+    │   │   └── storage_handler.go
+    │   ├── pv
+    │   │   ├── leaf_pv_controller.go
+    │   │   ├── oneway_pv_controller.go
+    │   │   └── root_pv_controller.go
+    │   └── pvc
+    │       ├── leaf_pvc_controller.go
+    │       ├── oneway_pvc_controller.go
+    │       └── root_pvc_controller.go
+    ├── extensions
+    │   └── daemonset
+    │       ├── constants.go
+    │       ├── daemonset_controller.go
+    │       ├── daemonset_mirror_controller.go
+    │       ├── distribute_controller.go
+    │       ├── host_daemon_controller.go
+    │       ├── pod_reflect_controller.go
+    │       ├── update.go
+    │       └── utils.go
+    ├── node-server
+    │   ├── api
+    │   │   ├── errdefs.go
+    │   │   ├── exec.go
+    │   │   ├── helper.go
+    │   │   ├── logs.go
+    │   │   └── remotecommand
+    │   │       ├── attach.go
+    │   │       ├── exec.go
+    │   │       ├── httpstream.go
+    │   │       └── websocket.go
+    │   └── server.go
+    └── utils
+        ├── leaf_model_handler.go
+        ├── leaf_resource_manager.go
+        └── rootcluster.go
+
+```

hack/build.sh

@@ -23,7 +23,7 @@ set -o pipefail
 REPO_ROOT=$(dirname "${BASH_SOURCE[0]}")/..
 source "${REPO_ROOT}/hack/util.sh"
 
-LDFLAGS="$(util::version_ldflags) ${LDFLAGS:-}"
+LDFLAGS="$(util::version_ldflags "${VERSION:-}") ${LDFLAGS:-}"
 
 function build_binary() {
   local -r target=$1
@@ -39,6 +39,7 @@ function build_binary() {
   done
 }
 
+
 function build_binary_for_platform() {
   local -r target=$1
   local -r platform=$2