test

.github/workflows/ci.yml

@@ -16,13 +16,13 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Install Ansible
-      run: sudo bash scripts/install/install_python_and_ansible.sh
+      run: sudo bash scripts/install/1.install_basic.sh
 
     - name: Install build
-      run: ansible-playbook -vvv scripts/install/ans_install_build.yml -i scripts/local_ansible_conf.ini
+      run: ansible-playbook -vvv scripts/install/2.ans_install_build.yml -i scripts/local_ansible_conf.ini
 
     - name: Build Locally
-      run: ansible-playbook -vvv scripts/deploy_single_node/ans_build.yml -i scripts/local_ansible_conf.ini
+      run: ansible-playbook -vvv scripts/deploy_single_node/1.ans_build.yml -i scripts/local_ansible_conf.ini
 
     - name: Integirty Test
       run: ansible-playbook -vvv scripts/ans_integity_test.yml -i scripts/local_ansible_conf.ini

scripts/build/1.1._ans_build_demo_apps.yml

@@ -0,0 +1,8 @@
+- name: Copy wasm
+  include_tasks: "../build/1.1.1_ans_build_demo_apps_each.yml"
+  loop:
+    - fn2
+    - word_count
+    - longchain
+- name: App needed data
+  shell: python3 ../build/1.2_gen_app_need_data.py

scripts/build/_ans_build_sys_with_apps.yml → scripts/build/1._ans_build_sys_with_apps.yml

@@ -13,37 +13,41 @@
     bash /tmp/compile.sh
     rm -f /tmp/compile.sh
 - name: Build the demo apps
-  shell: ansible-playbook -vvv build/ans_build_demo_apps.yml -i local_ansible_conf.ini
-  args:
-    chdir: ../
+  include_tasks: ../build/1.1._ans_build_demo_apps.yml
 # mkdir -p scripts/deploy_single_node/test_dir/files
 # mkdir -p scripts/deploy_single_node/test_dir/apps
 # cp scripts/deploy_single_node/node_config.yaml scripts/deploy_single_node/test_dir/files
-- name: Clear test_dir
-  shell: rm -rf test_dir
+- name: Clear pack
+  shell: rm -rf pack
 - name: Mkdirs
   file:
-    path: test_dir/{{ item }}
+    path: pack/test_dir/{{ item }}
     state: directory
   loop:
     - files
     - apps
   become: yes
-- name: Copy files
-  copy:
-    src: ../build/files
-    dest: test_dir
-    force: yes
+- name: Move files from build to cur dir
+  shell: mv files pack/test_dir
   become: yes
 - name: Copy node_config.yaml
   copy:
     src: node_config.yaml
-    dest: test_dir/files/node_config.yaml
+    dest: pack/test_dir/files/node_config.yaml
     force: yes
   become: yes
-- name: Copy Apps
+- name: Move apps from build to cur dir
+  shell: mv apps pack/test_dir
+  become: yes
+- name: Copy bin
   copy:
-    src: ../build/apps
-    dest: test_dir
+    src: ../../target/release/wasm_serverless
+    dest: pack
     force: yes
+  become: yes
+- name: Cat run_node.py
+  shell: cp ../build/run_node.py pack
+  become: yes
+- name: Permissions
+  shell: chmod -R 775 pack
   become: yes

scripts/build/run_node.py

@@ -0,0 +1,23 @@
+
+#     NODE_ID=$1
+#     wasm_serverless $NODE_ID test_dir
+
+import os
+import sys
+
+
+CUR_FPATH = os.path.abspath(__file__)
+CUR_FDIR = os.path.dirname(CUR_FPATH)
+os.chdir(CUR_FDIR)
+
+
+# export RUST_BACKTRACE=1
+os.environ['RUST_BACKTRACE'] = '1'
+# export RUST_LOG=info,wasm_serverless=debug
+os.environ['RUST_LOG'] = 'info,wasm_serverless=debug'
+
+
+#     NODE_ID=$1
+NODE_ID = sys.argv[1]
+#     wasm_serverless $NODE_ID test_dir
+os.system(f'./wasm_serverless {NODE_ID} test_dir')

scripts/deploy_cluster/.gitignore

@@ -2,4 +2,4 @@ promtail.yaml
 gen_ansible.ini
 gen_docker_stack.yml
 compose_*
-/test_dir
+/pack

scripts/deploy_cluster/_gen_each_node_compose.py → scripts/deploy_cluster/2.1_gen_each_node_compose.py

@@ -84,7 +84,15 @@ def promtail_config(lokiaddr):
         refresh_interval: 5s
         filters:
           - name: label
-            values: ["logging=promtail"]
+            values: ["log_promtail"]
+    relabel_configs:
+      - source_labels: ['__meta_docker_container_name']
+        regex: '/(.*)'
+        target_label: 'container'
+      - source_labels: ['__meta_docker_container_log_stream']
+        target_label: 'logstream'
+      - source_labels: ['__meta_docker_container_label_logging_jobname']
+        target_label: 'job'
     '''
     path=os.path.join(DEPLOY_CLUSTER_DIR, 'promtail.yaml')
     with open(path, 'w') as f:

scripts/deploy_cluster/2.ans_redeploy_cluster.sh

@@ -0,0 +1,2 @@
+export LANG=C.UTF-8
+ansible-playbook -vvv scripts/deploy_cluster/2.ans_redeploy_cluster.yml -i scripts/deploy_cluster/gen_ansible.ini

scripts/deploy_cluster/ans_redeploy_cluster.yml → scripts/deploy_cluster/2.ans_redeploy_cluster.yml

@@ -2,14 +2,16 @@
 - hosts: localhost
   tasks:
     - name: Install Build Related
-      include_tasks: ../install/_ans_install_build.yml
+      include_tasks: ../install/2._ans_install_build.yml
 
     - name: Build sys and demo apps
-      include_tasks: ../build/_ans_build_sys_with_apps.yml
-
+      include_tasks: ../build/1._ans_build_sys_with_apps.yml
 
     - name: Generate Docker Compose File
-      shell: python3 _gen_each_node_compose.py
+      shell: python3 2.1_gen_each_node_compose.py
+
+    - name: Move promtail.yaml to pack
+      shell: mv promtail.yaml pack/
 
     # Preparing for next zip
     - name: Remove Zip
@@ -19,7 +21,7 @@
 
     - name: Zip deploy_pack.zip (docker, scripts)
       # **/ means any path
-      command: "zip -r deploy_pack.zip scripts target/release/wasm_serverless -x **/random_words.txt"
+      command: "zip -r deploy_pack.zip scripts -x **/random_words.txt"
       args:
         chdir: ../..
 
@@ -32,13 +34,6 @@
     - name: Debug file path
       debug:
         var: src
-    - name: Make sure the directory exists
-      ansible.builtin.file:
-        path: "{{ item }}"
-        state: directory
-      become: yes
-      loop:
-        - /root/wasm_serverless_deploy
 
     - name: Stop & Remove container
       block:
@@ -51,63 +46,59 @@
         become: yes
         ignore_errors: yes
 
-    - name: Clear the target directory
-      shell: rm -rf /root/wasm_serverless_deploy/*
-      become: yes
-#         - /root/wasm_serverless_deploy/apps
-#         - /root/wasm_serverless_deploy/files
-    - name: Copy deploy_pack.zip to all nodes
-      copy:
-        src: ../../deploy_pack.zip
-        dest: /root/wasm_serverless_deploy/deploy_pack.zip
-      become: yes
-    - name: Copy docker-compose.yml to all nodes
-      copy:
-        src: compose_{{ ansible_host }}.yml
-        dest: /root/wasm_serverless_deploy/docker-compose.yml
-      become: yes
-
-    - name: Unzip deploy_pack.zip
-      unarchive:
-        src: /root/wasm_serverless_deploy/deploy_pack.zip
-        dest: /root/wasm_serverless_deploy
-        remote_src: yes
-      become: yes
-
-    - name: Copy big files
-      copy:
-        src: test_dir/files/random_words.txt
-        dest: /root/wasm_serverless_deploy/random_words.txt
-      become: yes
 
-    - name: Install Basic
-      shell: bash /root/wasm_serverless_deploy/scripts/install/install_python_and_ansible.sh
+    - name: Prepare remote files
+      block:
+      - name: Make sure the directory exists
+        ansible.builtin.file:
+          path: "{{ item }}"
+          state: directory
+        become: yes
+        loop:
+          - /root/wasm_serverless_deploy
+      - name: Clear the target directory
+        shell: rm -rf /root/wasm_serverless_deploy/*
+        become: yes
+      - name: Copy deploy_pack.zip to all nodes
+        copy:
+          src: ../../deploy_pack.zip
+          dest: /root/wasm_serverless_deploy/deploy_pack.zip
+        become: yes
+      - name: Copy docker-compose.yml to all nodes
+        copy:
+          src: compose_{{ ansible_host }}.yml
+          dest: /root/wasm_serverless_deploy/docker-compose.yml
+        become: yes
+      - name: Unzip deploy_pack.zip
+        unarchive:
+          src: /root/wasm_serverless_deploy/deploy_pack.zip
+          dest: /root/wasm_serverless_deploy
+          remote_src: yes
+        become: yes
+      - name: Remove zip
+        file:
+          path: /root/wasm_serverless_deploy/deploy_pack.zip
+          state: absent
+        become: yes
+      - name: Move built pack to root
+        shell: mv /root/wasm_serverless_deploy/scripts/deploy_cluster/pack/* /root/wasm_serverless_deploy/
+        become: yes
+      - name: Copy big files
+        copy:
+          src: pack/test_dir/files/random_words.txt
+          dest: /root/wasm_serverless_deploy/test_dir/files/random_words.txt
+        become: yes
+      - name: Permission dir
+        shell: chmod -R 775 /root/wasm_serverless_deploy
+        become: yes
+      - name: Debug deploy dir
+        shell: tree /root/wasm_serverless_deploy
 
-    - name: Remove zip
-      file:
-        path: /root/wasm_serverless_deploy/deploy_pack.zip
-        state: absent
-      become: yes
 
-    - name: Move test_dir to root
-      shell: mv /root/wasm_serverless_deploy/scripts/deploy_cluster/test_dir /root/wasm_serverless_deploy/
-      become: yes
-
-    - name: Move promtail.yaml to root
-      shell: mv /root/wasm_serverless_deploy/scripts/deploy_cluster/promtail.yaml /root/wasm_serverless_deploy/
-      become: yes
+    - name: Install Basic
+      shell: bash /root/wasm_serverless_deploy/scripts/install/1.install_basic.sh
 
-    - name: Move bin to test_dir
-      shell: mv /root/wasm_serverless_deploy/target/release/wasm_serverless /root/wasm_serverless_deploy/
-      become: yes
-
-    - name: Permission dir
-      shell: chmod -R 775 /root/wasm_serverless_deploy
-      become: yes
 
-    - name: Debug deploy dir
-      shell: tree /root/wasm_serverless_deploy
-
     - name: Docker Stuff
       block:
       - name: Install docker
@@ -142,7 +133,6 @@
           chdir: /root/wasm_serverless_deploy
         become: yes
 
-
       - name: Docker compose up
         command: docker-compose --compatibility up -d
         args:

scripts/deploy_single_node/.gitignore

@@ -1 +1 @@
-test_dir
+pack/

scripts/deploy_single_node/1.ans_build.sh

@@ -0,0 +1,3 @@
+export LANG=C.UTF-8
+source $HOME/.cargo/env
+ansible-playbook -vv scripts/deploy_single_node/1.ans_build.yml -i scripts/local_ansible_conf.ini

scripts/deploy_single_node/1.ans_build.yml

@@ -0,0 +1,32 @@
+---
+- hosts: localhost
+  tasks:
+    # - name: Build the application on the master node
+    #   become: yes
+    #   shell: |
+    #     cat > /tmp/compile.sh <<'END'
+    #       #!/bin/bash
+
+    #       echo $PATH
+    #       export PATH="/root/.cargo/bin/:$PATH"
+    #       export PATH="/root/.wasmedge/bin/:$PATH"
+    #       cargo build --release
+    #     END
+
+    #     bash /tmp/compile.sh
+    #     rm -f /tmp/compile.sh
+    # mkdir -p scripts/deploy_single_node/test_dir/files
+    # mkdir -p scripts/deploy_single_node/test_dir/apps
+    # cp scripts/deploy_single_node/node_config.yaml scripts/deploy_single_node/test_dir/files
+    - name: Clear pack
+      shell: rm -rf pack
+    - name: Install Build Related
+      include_tasks: ../install/2._ans_install_build.yml
+    - name: Build sys and demo apps
+      include_tasks: ../build/1._ans_build_sys_with_apps.yml  
+    - name: Copy node_config.yaml
+      copy:
+        src: node_config.yaml
+        dest: pack/test_dir/files/node_config.yaml
+        force: yes
+      become: yes