Merge branch 'main' into feature/path_group

.github/workflows/build_mac.yml

@@ -39,26 +39,40 @@ jobs:
 
       - name: Install tools and dependencies
         run: |
+
+          if [ ${{ matrix.build_type }} == "debug" ]; then
+            # Since the TEN runtime and TEN manager are a combination of C and
+            # Rust, to enable ASan, the ASan versions used by both the C and
+            # Rust compilation environments must be consistent. Therefore, it is
+            # necessary to lock the LLVM and Rust versions to enable ASan.
+            brew install llvm@18
+            rustup default nightly-2024-07-19
+          else
+            rustup default nightly
+          fi
+
           brew install tree
           pip3 install --use-pep517 python-dotenv jinja2
           go install golang.org/dl/go1.20.12@latest && go1.20.12 download
           go env -w GOFLAGS="-buildvcs=false"
-          rustup default nightly
           cargo install --force cbindgen
 
       - name: Build
         run: |
+          if [ ${{ matrix.build_type }} == "debug" ]; then
+            export PATH="/opt/homebrew/opt/llvm@18/bin:$PATH"
+          fi
           export PATH=$(pwd)/core/ten_gn:$PATH
           echo $PATH
-          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_test=false ten_enable_go_binding=${{ matrix.build_type == 'debug' && 'false' || 'true' }} ten_enable_package_manager=${{ matrix.build_type == 'debug' && 'false' || 'true' }} enable_sanitizer=false
+          tgn gen mac arm64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_test=false
           tgn build mac arm64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out
 
   build-x64:
     runs-on: macos-13
     strategy:
       matrix:
-        build_type: [debug, release]
+        build_type: [release]
     steps:
       - uses: actions/checkout@v4
         with:
@@ -76,17 +90,31 @@ jobs:
 
       - name: Install tools and dependencies
         run: |
+
+          if [ ${{ matrix.build_type }} == "debug" ]; then
+            # Since the TEN runtime and TEN manager are a combination of C and
+            # Rust, to enable ASan, the ASan versions used by both the C and
+            # Rust compilation environments must be consistent. Therefore, it is
+            # necessary to lock the LLVM and Rust versions to enable ASan.
+            brew install llvm@18
+            rustup default nightly-2024-07-19
+          else
+            rustup default nightly
+          fi
+
           brew install tree
           pip3 install --use-pep517 python-dotenv jinja2
           go install golang.org/dl/go1.20.12@latest && go1.20.12 download
           go env -w GOFLAGS="-buildvcs=false"
-          rustup default nightly
           cargo install --force cbindgen
 
       - name: Build
         run: |
+          if [ ${{ matrix.build_type }} == "debug" ]; then
+            export PATH="/usr/local/opt/llvm@18/bin:$PATH"
+          fi
           export PATH=$(pwd)/core/ten_gn:$PATH
           echo $PATH
-          tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_test=false ten_enable_go_binding=${{ matrix.build_type == 'debug' && 'false' || 'true' }} ten_enable_package_manager=${{ matrix.build_type == 'debug' && 'false' || 'true' }} enable_sanitizer=false
+          tgn gen mac x64 ${{ matrix.build_type }} -- log_level=1 enable_serialized_actions=true ten_enable_test=false
           tgn build mac x64 ${{ matrix.build_type }}
           tree -I 'gen|obj' out

build/common/rust/rust.gni

@@ -171,6 +171,14 @@ template("rust_test") {
       rebase_path(tg_timestamp_proxy_file),
     ]
 
+    if (defined(invoker.integration_test_output_name) &&
+        invoker.integration_test_output_name != "") {
+      args += [
+        "--integration-test-output-name",
+        invoker.integration_test_output_name,
+      ]
+    }
+
     args += [ "--build-type" ]
     if (is_debug) {
       args += [ "debug" ]
@@ -256,13 +264,6 @@ template("rust_test") {
       target,
     ]
 
-    if (defined(invoker.through_real_test) && invoker.through_real_test) {
-      args += [
-        "--through-real-test",
-        "True",
-      ]
-    }
-
     forward_variables_from(invoker,
                            [
                              "deps",

build/common/rust/rust_build_tests.py

@@ -5,76 +5,88 @@
 # Refer to the "LICENSE" file in the root directory for more information.
 #
 import argparse
+import json
 import sys
 import os
-import platform
 from build.scripts import cmd_exec, fs_utils, timestamp_proxy
 
 
 class ArgumentInfo(argparse.Namespace):
     def __init__(self):
         super().__init__()
-        self.through_real_test: bool
+        self.no_run: bool
         self.project_path: str
         self.build_type: str
         self.target_path: str
         self.env: list[str]
         self.log_level: int
         self.test_output_dir: str
         self.tg_timestamp_proxy_file: str | None = None
+        self.integration_test_output_name: str | None = None
+
+
+def get_crate_test_output_name(log_level: int) -> str:
+    cmd = ["cargo", "metadata", "--no-deps", "--format-version", "1"]
+    returncode, output = cmd_exec.run_cmd(cmd, log_level)
+    if returncode:
+        raise Exception(f"Failed to get crate name: {output}")
+
+    # Get the last line of the output, as there might be some note or warning
+    # messages from cargo.
+    output = output.splitlines()[-1]
+    metadata = json.loads(output)
+    for target in metadata["packages"][0]["targets"]:
+        if target["test"]:
+            return target["name"]
+
+    raise Exception("Failed to get crate name from targets.")
 
 
 def copy_test_binaries(
-    through_real_test: bool,
     source_dir: str,
+    base_name: str,
     output_dir: str,
+    target_name: str,
     log_level: int = 0,
 ):
     if log_level > 0:
-        print(f"Copying test binaries from {source_dir} to {output_dir}")
+        print(
+            f"Copying test binary {base_name} from {source_dir} to {output_dir}"
+        )
 
-    if through_real_test:
-        if platform.system() == "Windows":
-            executable_name = "tman_test.exe"
-        else:
-            executable_name = "tman_test"
+    for f in os.listdir(source_dir):
+        is_executable = os.access(os.path.join(source_dir, f), os.X_OK)
+        if not is_executable:
+            continue
+
+        # The file name generated by `cargo build` always has a hash suffix, ex:
+        # <library_name>-<hash>.<extension>. We need to remove the hash suffix.
+
+        # We have to deal with the extension of the executable file, as if the
+        # `crate-type` is `cdylib`, the extension is `.so` on linux, which is
+        # not what we want. The extension is not present on linux or macos, and
+        # the extension is '.exe' on windows.
+        extension = os.path.splitext(f)[1]
+        if extension != "" and extension != ".exe":
+            continue
+
+        binary_name = f[: f.rindex("-") if "-" in f else -1]
+        if binary_name != base_name:
+            continue
+
+        dest = target_name + extension
 
         fs_utils.copy_file(
-            os.path.join(source_dir, executable_name),
-            os.path.join(output_dir, executable_name),
+            os.path.join(source_dir, f),
+            os.path.join(output_dir, dest),
             True,
         )
-    else:
-        for f in os.listdir(source_dir):
-            is_executable = os.access(os.path.join(source_dir, f), os.X_OK)
-            if not is_executable:
-                continue
-
-            # The file name generated by `cargo build` always has a hash suffix,
-            # ex: <library_name>-<hash>.<extension>. We need to remove the hash
-            # suffix.
-
-            # We have to deal with the extension of the executable file, as if
-            # the `crate-type` is `cdylib`, the extension is `.so` on linux,
-            # which is not what we want. The extension is not present on linux
-            # or macos, and the extension is '.exe' on windows.
-            extension = os.path.splitext(f)[1]
-            if extension != "" and extension != ".exe":
-                continue
-
-            ff = f[: f.rindex("-") if "-" in f else -1] + extension
-
-            fs_utils.copy_file(
-                os.path.join(source_dir, f),
-                os.path.join(output_dir, ff),
-                True,
-            )
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
 
-    parser.add_argument("--through-real-test", type=bool, default=False)
+    parser.add_argument("--no-run", type=bool, default=True)
     parser.add_argument("--project-path", type=str, required=True)
     parser.add_argument("--manifest-path", type=str, required=False)
     parser.add_argument("--build-type", type=str, required=True)
@@ -88,6 +100,9 @@ def copy_test_binaries(
     parser.add_argument(
         "--tg-timestamp-proxy-file", type=str, default="", required=False
     )
+    parser.add_argument(
+        "--integration-test-output-name", type=str, required=False
+    )
 
     arg_info = ArgumentInfo()
     args = parser.parse_args(namespace=arg_info)
@@ -108,24 +123,30 @@ def copy_test_binaries(
     try:
         os.chdir(args.project_path)
 
-        if args.through_real_test:
+        # cargo build --tests: only compile the test source files, without
+        # running the test cases.
+        # cargo test: compile the test source files and run the test cases.
+        #
+        # `cargo build --tests` or `cargo test --no-run` will not trigger the
+        # `runner` script in .cargo/config.toml.
+        if args.no_run:
             cmd = [
                 "cargo",
-                "test",
+                "build",
                 "--target-dir",
                 args.target_path,
                 "--target",
                 args.target,
+                "--tests",
             ]
         else:
             cmd = [
                 "cargo",
-                "build",
+                "test",
                 "--target-dir",
                 args.target_path,
                 "--target",
                 args.target,
-                "--tests",
             ]
 
         if args.build_type == "release":
@@ -134,23 +155,45 @@ def copy_test_binaries(
         returncode, logs = cmd_exec.run_cmd(cmd, args.log_level)
         if returncode:
             raise Exception(f"Failed to build rust tests: {logs}")
+        else:
+            print(logs)
 
-        # The output of the dependencies will be in <target_path>/<build_type>
-        # /deps, while the output of the tests will be in <target_path>/<target>
-        # /<build_type>/deps.
+        # The prefix of the unit test binary is the crate name (i.e., the
+        # 'name' in [[bin]] or [[lib]] in a crate).
+        unit_test_output_name = get_crate_test_output_name(args.log_level)
+
+        # The output of the tests will be in
+        # <target_path>/<target>/<build_type>/deps
         if args.test_output_dir != "":
+            # Copy the unit test binary.
             copy_test_binaries(
-                args.through_real_test,
                 os.path.join(
                     args.target_path,
                     args.target,
                     args.build_type,
                     "deps",
                 ),
+                unit_test_output_name,
                 args.test_output_dir,
+                "unit_test",
                 args.log_level,
             )
 
+            if args.integration_test_output_name:
+                # Copy the integration test binary.
+                copy_test_binaries(
+                    os.path.join(
+                        args.target_path,
+                        args.target,
+                        args.build_type,
+                        "deps",
+                    ),
+                    args.integration_test_output_name,
+                    args.test_output_dir,
+                    "integration_test",
+                    args.log_level,
+                )
+
         # Success to build the app, update the stamp file to represent this
         # fact.
         timestamp_proxy.touch_timestamp_proxy_file(args.tg_timestamp_proxy_file)

core/src/ten_manager/BUILD.gn

@@ -25,8 +25,7 @@ if (ten_enable_package_manager) {
       utils_static_lib_path = rebase_path("${root_gen_dir}/core/src/ten_utils")
       env += [ "TEN_UTILS_LIBRARY_PATH=${utils_static_lib_path}" ]
 
-      test_output_dir = "${root_out_dir}/tests/standalone"
-      through_real_test = true
+      test_output_dir = "${root_out_dir}/tests/standalone/ten_manager"
 
       deps = [
         "//core/src/ten_rust:ten_rust_static_lib",