test(deploy_cluster): fit deploy_cluster and deploy_monitor to latest…

… scripts

.github/workflows/ci.yml

@@ -16,13 +16,13 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Install Ansible
-      run: sudo bash scripts/install/install_python_and_ansible.sh
+      run: sudo bash scripts/install/1.install_basic.sh
 
     - name: Install build
-      run: ansible-playbook -vvv scripts/install/ans_install_build.yml -i scripts/local_ansible_conf.ini
+      run: ansible-playbook -vvv scripts/install/2.ans_install_build.yml -i scripts/local_ansible_conf.ini
 
     - name: Build Locally
-      run: ansible-playbook -vvv scripts/deploy_single_node/ans_build.yml -i scripts/local_ansible_conf.ini
+      run: ansible-playbook -vvv scripts/deploy_single_node/1.ans_build.yml -i scripts/local_ansible_conf.ini
 
     - name: Integirty Test
       run: ansible-playbook -vvv scripts/ans_integity_test.yml -i scripts/local_ansible_conf.ini

scripts/build/1.1._ans_build_demo_apps.yml

@@ -0,0 +1,8 @@
+- name: Copy wasm
+  include_tasks: "../build/1.1.1_ans_build_demo_apps_each.yml"
+  loop:
+    - fn2
+    - word_count
+    - longchain
+- name: App needed data
+  shell: python3 ../build/1.2_gen_app_need_data.py

scripts/build/1._ans_build_sys_with_apps.yml

@@ -0,0 +1,53 @@
+- name: Build the main system on the master node
+  become: yes
+  shell: |
+    cat > /tmp/compile.sh <<'END'
+      #!/bin/bash
+
+      echo $PATH
+      export PATH="/root/.cargo/bin/:$PATH"
+      export PATH="/root/.wasmedge/bin/:$PATH"
+      cargo build --release
+    END
+
+    bash /tmp/compile.sh
+    rm -f /tmp/compile.sh
+- name: Build the demo apps
+  include_tasks: ../build/1.1._ans_build_demo_apps.yml
+# mkdir -p scripts/deploy_single_node/test_dir/files
+# mkdir -p scripts/deploy_single_node/test_dir/apps
+# cp scripts/deploy_single_node/node_config.yaml scripts/deploy_single_node/test_dir/files
+- name: Clear pack
+  shell: rm -rf pack
+- name: Mkdirs
+  file:
+    path: pack/test_dir/{{ item }}
+    state: directory
+  loop:
+    - files
+    - apps
+  become: yes
+- name: Move files from build to cur dir
+  shell: mv files pack/test_dir
+  become: yes
+- name: Copy node_config.yaml
+  copy:
+    src: node_config.yaml
+    dest: pack/test_dir/files/node_config.yaml
+    force: yes
+  become: yes
+- name: Move apps from build to cur dir
+  shell: mv apps pack/test_dir
+  become: yes
+- name: Copy bin
+  copy:
+    src: ../../target/release/wasm_serverless
+    dest: pack
+    force: yes
+  become: yes
+- name: Cat run_node.py
+  shell: cp ../build/run_node.py pack
+  become: yes
+- name: Permissions
+  shell: chmod -R 775 pack
+  become: yes

scripts/build/run_node.py

@@ -0,0 +1,23 @@
+
+#     NODE_ID=$1
+#     wasm_serverless $NODE_ID test_dir
+
+import os
+import sys
+
+
+CUR_FPATH = os.path.abspath(__file__)
+CUR_FDIR = os.path.dirname(CUR_FPATH)
+os.chdir(CUR_FDIR)
+
+
+# export RUST_BACKTRACE=1
+os.environ['RUST_BACKTRACE'] = '1'
+# export RUST_LOG=info,wasm_serverless=debug
+os.environ['RUST_LOG'] = 'info,wasm_serverless=debug'
+
+
+#     NODE_ID=$1
+NODE_ID = sys.argv[1]
+#     wasm_serverless $NODE_ID test_dir
+os.system(f'./wasm_serverless {NODE_ID} test_dir')

scripts/deploy_cluster/.gitignore

@@ -1,3 +1,5 @@
+promtail.yaml
 gen_ansible.ini
 gen_docker_stack.yml
-compose_*
+compose_*
+/pack

scripts/deploy_cluster/1.1.gen_ansible_conf.py

@@ -0,0 +1,86 @@
+import os
+import yaml
+import argparse
+import sys
+import pexpect
+
+PASSWORD="aaaaa"
+
+def run_cmd(cmd):
+    print("> "+cmd)
+    # if cmd.startswith("ssh") or cmd.startswith("scp"):
+    #     # 创建spawn对象
+    #     child = pexpect.spawn(cmd, encoding='utf-8',logfile=sys.stdout)
+
+    #     # 匹配密码提示，然后发送密码
+    #     child.expect('password:')
+    #     child.sendline(PASSWORD)
+
+    #     # 在这里可以继续与SSH会话进行交互
+    #     # 例如，可以发送其他命令
+
+    #     # 等待命令执行完成
+    #     try:
+    #         child.expect(pexpect.EOF)
+    #     except:
+    #         pass
+    #     child.close()
+    #     # 打印输出
+    #     # print(child.before)
+    # else:
+    os.system(cmd)
+
+
+def read_yaml(f):
+    # parse
+    import ruamel.yaml
+    yaml = ruamel.yaml.YAML(typ='rt')
+    parsed_data = yaml.load(f)
+
+    return parsed_data
+
+def entry():
+    # read cluster-nodes.yml
+    with open('scripts/deploy_cluster/node_config.yaml', 'r') as f:
+        # run_cmd("scripts/install/install_ansible.sh")
+
+        # write to gen_ansible.ini
+        ansible="[web]\n"
+
+        # # gen ssh key if not exist
+        # if not os.path.exists("/root/.ssh/id_rsa"):
+        #     run_cmd("ssh-keygen -t rsa -b 2048")
+
+        cluster_nodes = read_yaml(f)
+        appeared_node={}
+        for nid in cluster_nodes["nodes"]:
+            node=cluster_nodes["nodes"][nid]
+            ip=node["addr"].split(":")[0]
+            port=node["addr"].split(":")[1]
+            # id=node["id"]
+
+            if ip not in appeared_node:
+                ansible+="webserver{} ansible_host={} ansible_user=root\n".format(nid,ip)
+                appeared_node[ip]=1
+
+            # run_cmd("ssh root@{} 'apt install python'".format(ip))
+            # run_cmd("ssh-copy-id root@{}".format(ip))
+
+        # write to gen_ansible.ini
+        with open("scripts/deploy_cluster/gen_ansible.ini","w") as f:
+            f.write(ansible)
+
+
+        # with open("gen_ansible.cfg","w") as f:
+        #     f.write(
+        #         "[defaults]\n"+\
+        #         "inventory = ./gen_ansible.ini\n"+\
+        #         "remote_user = root\n"+\
+        #         "private_key_file = /root/.ssh/id_rsa\n"+\
+        #         "host_key_checking = False"
+        #     )
+
+        # run ansible
+        run_cmd("ansible -i scripts/deploy_cluster/gen_ansible.ini -m ping all")
+
+entry()

scripts/deploy_cluster/2.1_gen_each_node_compose.py

@@ -0,0 +1,120 @@
+import yaml
+import os
+
+# Get the directory of the current script
+DEPLOY_CLUSTER_DIR = os.path.dirname(os.path.abspath(__file__))
+NODE_CONFIG = os.path.join(DEPLOY_CLUSTER_DIR, 'node_config.yaml')
+
+
+def read_yaml(file_path):
+    with open(file_path, 'r') as file:
+        data = yaml.safe_load(file)
+    return data
+
+
+def generate_docker_compose(ip, nodes):
+    services =yaml.safe_load('''
+        promtail:
+            image: "grafana/promtail"
+            volumes:
+                - /var/lib/docker/containers:/var/lib/docker/containers:ro
+                - /var/run/docker.sock:/var/run/docker.sock
+                - /root/wasm_serverless_deploy/promtail.yaml:/etc/promtail/promtail.yaml
+            command: -config.file=/etc/promtail/promtail.yaml
+        ''')
+
+    for key, node in nodes.items():
+        service_name = f"node{key}"
+
+        external_port1 = int(node['addr'].split(':')[-1])
+        external_port2 = external_port1 + 1
+        external_ip = node['addr'].split(':')[0]
+
+        services[service_name] = {
+            'image': 'wasm_serverless:v1',
+            'ports': [f"{external_ip}:{external_port1}:{external_port1}/udp", f"{external_port2}:{external_port2}"],
+            'deploy':{
+                'resources':{
+                    'limits':{
+                        'memory': '6G'
+                    }
+                }
+            },
+            'volumes': [
+                '/root/wasm_serverless_deploy:/etc/wasm_serverless',
+            ],
+            'environment': {
+                'WASM_SERVERLESS_NODEID': key
+            },
+            'privileged': True, # for tc control,
+            'logging': {
+                # 设置日志驱动程序和选项
+                'driver': "json-file",
+                'options':{
+                    'max-size': "10m",
+                    'max-file': "5"
+                }
+            },
+            'labels':["log_promtail"]
+        }
+
+
+    compose_data = {'version': '3', 'services': services}
+    compose_file_name = os.path.join(DEPLOY_CLUSTER_DIR, f"compose_{ip}.yml")
+
+    with open(compose_file_name, 'w') as file:
+        yaml.dump(compose_data, file, default_flow_style=False)
+
+def promtail_config(lokiaddr):
+    PROMTAIL_CONFIG = f'''
+server:
+  http_listen_port: 9080
+  grpc_listen_port: 0
+
+positions:
+  filename: /tmp/positions.yaml
+
+clients:
+  - url: http://{lokiaddr}/loki/api/v1/push
+
+scrape_configs:
+  - job_name: flog_scrape
+    docker_sd_configs:
+      - host: unix:///var/run/docker.sock
+        refresh_interval: 5s
+        filters:
+          - name: label
+            values: ["log_promtail"]
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: 'logstream'
+      - source_labels: ['__meta_docker_container_label_logging_jobname']
+        target_label: 'job'
+    '''
+    path=os.path.join(DEPLOY_CLUSTER_DIR, 'promtail.yaml')
+    with open(path, 'w') as f:
+        f.write(PROMTAIL_CONFIG)
+
+
+
+def main():
+    data = read_yaml(NODE_CONFIG)
+
+    grouped_nodes = {}
+    for key, node in data['nodes'].items():
+        ip = node['addr'].split(':')[0]
+        if ip not in grouped_nodes:
+            grouped_nodes[ip] = {}
+        grouped_nodes[ip][key] = node
+
+    for ip, nodes in grouped_nodes.items():
+        generate_docker_compose(ip, nodes)
+
+    promtail_config(data['loki']['addr'])
+
+
+if __name__ == "__main__":
+    main()

scripts/deploy_cluster/2.ans_redeploy_cluster.sh

@@ -0,0 +1,2 @@
+export LANG=C.UTF-8
+ansible-playbook -vvv scripts/deploy_cluster/2.ans_redeploy_cluster.yml -i scripts/deploy_cluster/gen_ansible.ini