From f29214105a4b938d48de410e62eb1cc7030ddbd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Thu, 23 Nov 2023 14:49:22 +0100 Subject: [PATCH 01/35] Replace make with pip invoke Fixes #701 WIP --- package.json | 21 +-- worker/scripts/get-dep.sh | 2 +- worker/scripts/get-pip-invoke.sh | 7 + worker/tasks.py | 235 +++++++++++++++++++++++++++++++ 4 files changed, 254 insertions(+), 11 deletions(-) create mode 100755 worker/scripts/get-pip-invoke.sh create mode 100644 worker/tasks.py diff --git a/package.json b/package.json index 625cda38ed..e9d9a87b9f 100644 --- a/package.json +++ b/package.json @@ -21,23 +21,24 @@ "types": "node/lib/index.d.ts", "files": [ "node/lib", - "worker/deps/libwebrtc", + "npm-scripts.mjs", + "worker/tasks.py", + "worker/Makefile", + "worker/meson.build", + "worker/meson_options.txt", + "worker/src", + "worker/include", "worker/fbs", - "worker/fuzzer/include", + "worker/test/src", + "worker/test/include", "worker/fuzzer/src", - "worker/include", + "worker/fuzzer/include", "worker/scripts/*.mjs", "worker/scripts/*.json", "worker/scripts/*.py", "worker/scripts/*.sh", - "worker/src", "worker/subprojects/*.wrap", - "worker/test/include", - "worker/test/src", - "worker/Makefile", - "worker/meson.build", - "worker/meson_options.txt", - "npm-scripts.mjs" + "worker/deps/libwebrtc" ], "engines": { "node": ">=16" diff --git a/worker/scripts/get-dep.sh b/worker/scripts/get-dep.sh index d538ae8606..5d6fced81f 100755 --- a/worker/scripts/get-dep.sh +++ b/worker/scripts/get-dep.sh @@ -7,7 +7,7 @@ DEP=$1 current_dir_name=${WORKER_PWD##*/} if [ "${current_dir_name}" != "worker" ] ; then - echo ">>> [ERROR] $(basename $0) must be called from mediasoup/worker/ directory" >&2 + echo ">>> [ERROR] $(basename $0) must be called from mediasoup/worker directory" >&2 exit 1 fi diff --git a/worker/scripts/get-pip-invoke.sh b/worker/scripts/get-pip-invoke.sh new file mode 100755 index 0000000000..b420733e62 --- /dev/null +++ b/worker/scripts/get-pip-invoke.sh @@ -0,0 +1,7 @@ +#!/usr/bin/env bash + +set -e + +PYTHON=${PYTHON:="python3"} + +${PYTHON} -m pip install invoke diff --git a/worker/tasks.py b/worker/tasks.py new file mode 100644 index 0000000000..ed1af6e564 --- /dev/null +++ b/worker/tasks.py @@ -0,0 +1,235 @@ +# This is a tasks.py file for the pip invoke package: https://docs.pyinvoke.org/. +# +# It's a replacement of our Makefile with same tasks. Obviously it requires +# having the pip invoke package installed, however I think we could just +# include it in the mediasoup repo in the worker/pip-invoke folder and make +# npm-scripts.mjs invoke its executable in worker/pip-invoke/bin/invoke (same +# for Rust). However that's hard since we also need to later override PYTHONPATH +# env for pip invoke module to be found. So... +# +# Perhaps it's easier if we run worker/scripts/get-pip-invoke.sh during the +# installation. That script installs pip invoke package in default pip location. +# +# Or we can set PYTHONPATH env within npm-scripts.mjs (and Rust) and also in the +# Makefile, so we don't remove it and instead make it behave as a proxy to invoke. +# +# Let's see. + + +import sys; +import os; +import inspect; +import shutil; +from invoke import task; + + +MEDIASOUP_BUILDTYPE = os.getenv('MEDIASOUP_BUILDTYPE') or 'Release'; + +WORKER_DIR = os.path.dirname(os.path.abspath( + inspect.getframeinfo(inspect.currentframe()).filename +)); +OUT_DIR = os.getenv('OUT_DIR') or f'{WORKER_DIR}/out'; +INSTALL_DIR = os.getenv('INSTALL_DIR') or f'{OUT_DIR}/{MEDIASOUP_BUILDTYPE}'; +BUILD_DIR = os.getenv('BUILD_DIR') or f'{INSTALL_DIR}/build'; +PIP_DIR = f'{OUT_DIR}/pip'; + +PYTHON = os.getenv('PYTHON') or sys.executable; +MESON = os.getenv('MESON') or f'{PIP_DIR}/bin/meson'; +MESON_VERSION = os.getenv('MESON_VERSION') or '1.2.1'; +# MESON_ARGS can be used to provide extra configuration parameters to meson, +# such as adding defines or changing optimization options. For instance, use +# `MESON_ARGS="-Dms_log_trace=true -Dms_log_file_line=true" npm i` to compile +# worker with tracing and enabled. +# NOTE: On Windows make sure to add `--vsenv` or have MSVS environment already +# active if you override this parameter. +# TODO: Replicate this: +# # Activate VS environment on Windows by default. +# ifeq ($(OS),Windows_NT) +# ifeq ($(MESON_ARGS),"") +# MESON_ARGS = $(subst $\",,"--vsenv") +# endif +# endif +MESON_ARGS = ''; +# Let's use a specific version of ninja to avoid buggy version 1.11.1: +# https://mediasoup.discourse.group/t/partly-solved-could-not-detect-ninja-v1-8-2-or-newer/ +# https://github.com/ninja-build/ninja/issues/2211 +# https://github.com/ninja-build/ninja/issues/2212 +NINJA_VERSION = os.getenv('NINJA_VERSION') or '1.10.2.4'; + +# TODO: Remove. +print('WORKER_DIR:', WORKER_DIR); +print('OUT_DIR:', OUT_DIR); +print('INSTALL_DIR:', INSTALL_DIR); +print('BUILD_DIR:', BUILD_DIR); +print('PIP_DIR:', PIP_DIR); +print('PYTHON:', PYTHON); +print('MESON:', MESON); + + +@task +def meson_ninja(ctx): + """ + Installs meson and ninja (also updates Python pip and setuptools packages) + """ + + if os.path.isdir(PIP_DIR): + return; + + # Updated pip and setuptools are needed for meson. + # `--system` is not present everywhere and is only needed as workaround for + # Debian-specific issue (copied from https://github.com/gluster/gstatus/pull/33), + # fallback to command without `--system` if the first one fails. + try: + ctx.run( + f'{PYTHON} -m pip install --system --target={PIP_DIR} pip setuptools', + echo=True, + hide=True + ); + except: + ctx.run( + f'{PYTHON} -m pip install --target={PIP_DIR} pip setuptools', + echo=True + ); + + # Workaround for NixOS and Guix that don't work with pre-built binaries, see: + # https://github.com/NixOS/nixpkgs/issues/142383. + PIP_BUILD_BINARIES = ''; + + if os.path.isfile('/etc/NIXOS') or os.path.isdir('/etc/guix'): + PIP_BUILD_BINARIES = '--no-binary :all:'; + + # Install meson and ninja using pip into our custom location, so we don't + # depend on system-wide installation. + ctx.run( + f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} {PIP_BUILD_BINARIES} meson=={MESON_VERSION} ninja=={NINJA_VERSION}', + echo=True + ); + + +@task(pre=[meson_ninja]) +def setup(ctx): + """ + Runs meson setup + """ + + # We try to call `--reconfigure` first as a workaround for this issue: + # https://github.com/ninja-build/ninja/issues/1997 + if MEDIASOUP_BUILDTYPE == 'Release': + try: + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', + echo=True + ); + except: + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', + echo=True + ); + elif MEDIASOUP_BUILDTYPE == 'Debug': + try: + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', + echo=True + ); + except: + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', + echo=True + ); + else: + try: + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', + echo=True + ); + except: + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', + echo=True + ); + + +@task +def clean(ctx): + """ + Cleans the installation directory + """ + + try: + shutil.rmtree(INSTALL_DIR); + except: + pass; + + +@task +def clean_build(ctx): + """ + Cleans the build directory + """ + + try: + shutil.rmtree(BUILD_DIR); + except: + pass; + + +@task +def clean_pip(ctx): + """ + Cleans the local pip directory + """ + + try: + shutil.rmtree(PIP_DIR); + except: + pass; + + +@task(pre=[meson_ninja]) +def clean_subprojects(ctx): + """ + Cleans meson subprojects + """ + + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} subprojects purge --include-cache --confirm', + echo=True + ); + + +@task +def clean_all(ctx): + """ + Cleans meson subprojects and all installed/built artificats + """ + + with ctx.cd(WORKER_DIR): + try: + ctx.run( + f'{MESON} subprojects purge --include-cache --confirm', + echo=True + ); + except: + pass; + + try: + shutil.rmtree(OUT_DIR); + except: + pass; + + +@task(pre=[meson_ninja]) +def foo(ctx): + """ + Foo things (until I figure out how to define a default task) + """ + + print('foo!'); + From 5c3ba8731844cae66efc3d0eca434c60594e94ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Thu, 23 Nov 2023 16:40:17 +0100 Subject: [PATCH 02/35] Reorder "files" in package.json (folders first) --- package.json | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index e9d9a87b9f..9aa28d4a5b 100644 --- a/package.json +++ b/package.json @@ -21,11 +21,6 @@ "types": "node/lib/index.d.ts", "files": [ "node/lib", - "npm-scripts.mjs", - "worker/tasks.py", - "worker/Makefile", - "worker/meson.build", - "worker/meson_options.txt", "worker/src", "worker/include", "worker/fbs", @@ -38,7 +33,12 @@ "worker/scripts/*.py", "worker/scripts/*.sh", "worker/subprojects/*.wrap", - "worker/deps/libwebrtc" + "worker/deps/libwebrtc", + "worker/tasks.py", + "worker/Makefile", + "worker/meson.build", + "worker/meson_options.txt", + "npm-scripts.mjs" ], "engines": { "node": ">=16" From 76f7195107cf69cd664c82aacaae551e0cb80a75 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Thu, 23 Nov 2023 18:43:55 +0100 Subject: [PATCH 03/35] WIP --- TODO_INVOKE.md | 11 +++ worker/Makefile | 18 ++-- worker/scripts/cpu_cores.sh | 2 +- worker/tasks.py | 191 +++++++++++++++++++++++++++++++++--- 4 files changed, 197 insertions(+), 25 deletions(-) create mode 100644 TODO_INVOKE.md diff --git a/TODO_INVOKE.md b/TODO_INVOKE.md new file mode 100644 index 0000000000..31c155f038 --- /dev/null +++ b/TODO_INVOKE.md @@ -0,0 +1,11 @@ +# TODO pip invoke + +- I've renamed `MEDIASOUP_OUT_DIR` to `OUT_DIR`, so check if set/used somewhere else. + +- `build.rs`: Remvoe the `MEDIASOUP_OUT_DIR` env stuff. + +- Remove `getmake.py` and related stuff. + +- When running meson commands we are loosing colored output, perhaps some invoke `context.run()` option is needed. Reported here: https://github.com/pyinvoke/invoke/discussions/976 + +- Every time I run `invoke test` or `make test` the whole worker is built from scratch :( This may happen because we run `flatc` task anyway in `tasks.py` (maybe a TODO), but why when using `make test`??? diff --git a/worker/Makefile b/worker/Makefile index 820bac829c..1bf97aebf6 100644 --- a/worker/Makefile +++ b/worker/Makefile @@ -4,8 +4,8 @@ # We need Python 3 here. PYTHON ?= $(shell command -v python3 2> /dev/null || echo python) -ROOT_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) -CORES ?= $(shell ${ROOT_DIR}/scripts/cpu_cores.sh || echo 4) +WORKER_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) +NUM_CORES ?= $(shell ${WORKER_DIR}/scripts/cpu_cores.sh || echo 4) MEDIASOUP_OUT_DIR ?= $(shell pwd)/out # Controls build types, `Release` and `Debug` are presets optimized for those # use cases. Other build types are possible too, but they are not presets and @@ -13,7 +13,7 @@ MEDIASOUP_OUT_DIR ?= $(shell pwd)/out # Check the meaning of useful macros in the `worker/include/Logger.hpp` header # file if you want to enable tracing or other debug information. MEDIASOUP_BUILDTYPE ?= Release -LCOV = ./deps/lcov/bin/lcov +LCOV = $(WORKER_DIR)/deps/lcov/bin/lcov DOCKER ?= docker PIP_DIR = $(MEDIASOUP_OUT_DIR)/pip INSTALL_DIR ?= $(MEDIASOUP_OUT_DIR)/$(MEDIASOUP_BUILDTYPE) @@ -203,12 +203,12 @@ update-wrap-file: meson-ninja mediasoup-worker: setup flatc ifeq ($(MEDIASOUP_WORKER_BIN),) - $(MESON) compile -C $(BUILD_DIR) -j $(CORES) mediasoup-worker + $(MESON) compile -C $(BUILD_DIR) -j $(NUM_CORES) mediasoup-worker $(MESON) install -C $(BUILD_DIR) --no-rebuild --tags mediasoup-worker endif libmediasoup-worker: setup flatc - $(MESON) compile -C $(BUILD_DIR) -j $(CORES) libmediasoup-worker + $(MESON) compile -C $(BUILD_DIR) -j $(NUM_CORES) libmediasoup-worker $(MESON) install -C $(BUILD_DIR) --no-rebuild --tags libmediasoup-worker flatc: setup @@ -224,7 +224,7 @@ format: $(NPM) run format --prefix scripts/ test: setup flatc - $(MESON) compile -C $(BUILD_DIR) -j $(CORES) mediasoup-worker-test + $(MESON) compile -C $(BUILD_DIR) -j $(NUM_CORES) mediasoup-worker-test $(MESON) install -C $(BUILD_DIR) --no-rebuild --tags mediasoup-worker-test ifeq ($(OS),Windows_NT) # On Windows lcov doesn't work (at least not yet) and we need to add `.exe` to @@ -236,7 +236,7 @@ else endif test-asan: setup flatc - $(MESON) compile -C $(BUILD_DIR) -j $(CORES) mediasoup-worker-test-asan + $(MESON) compile -C $(BUILD_DIR) -j $(NUM_CORES) mediasoup-worker-test-asan $(MESON) install -C $(BUILD_DIR) --no-rebuild --tags mediasoup-worker-test-asan ASAN_OPTIONS=detect_leaks=1 $(BUILD_DIR)/mediasoup-worker-test-asan --invisibles --use-colour=yes $(MEDIASOUP_TEST_TAGS) @@ -246,12 +246,12 @@ tidy: -clang-apply-replacements-binary=./scripts/node_modules/.bin/clang-apply-replacements \ -header-filter="(Channel/**/*.hpp|DepLibSRTP.hpp|DepLibUV.hpp|DepLibWebRTC.hpp|DepOpenSSL.hpp|DepUsrSCTP.hpp|LogLevel.hpp|Logger.hpp|MediaSoupError.hpp|RTC/**/*.hpp|Settings.hpp|Utils.hpp|Worker.hpp|common.hpp|handles/**/*.hpp)" \ -p=$(BUILD_DIR) \ - -j=$(CORES) \ + -j=$(NUM_CORES) \ -checks=$(MEDIASOUP_TIDY_CHECKS) \ -quiet fuzzer: setup flatc - $(MESON) compile -C $(BUILD_DIR) -j $(CORES) mediasoup-worker-fuzzer + $(MESON) compile -C $(BUILD_DIR) -j $(NUM_CORES) mediasoup-worker-fuzzer $(MESON) install -C $(BUILD_DIR) --no-rebuild --tags mediasoup-worker-fuzzer fuzzer-run-all: diff --git a/worker/scripts/cpu_cores.sh b/worker/scripts/cpu_cores.sh index b5cdde81d0..4419b6f51e 100755 --- a/worker/scripts/cpu_cores.sh +++ b/worker/scripts/cpu_cores.sh @@ -9,7 +9,7 @@ case "${OS}" in Linux*) NUM_CORES=$(nproc);; Darwin*|FreeBSD) NUM_CORES=$(sysctl -n hw.ncpu);; MINGW*) NUM_CORES=$NUMBER_OF_PROCESSORS;; - *) NUM_CORES=1;; + *) NUM_CORES=4;; esac if [ -n "${MEDIASOUP_MAX_CORES}" ]; then diff --git a/worker/tasks.py b/worker/tasks.py index ed1af6e564..2996955e8d 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -20,6 +20,8 @@ import os; import inspect; import shutil; +import subprocess; +import re; from invoke import task; @@ -33,6 +35,10 @@ BUILD_DIR = os.getenv('BUILD_DIR') or f'{INSTALL_DIR}/build'; PIP_DIR = f'{OUT_DIR}/pip'; +NUM_CORES = re.findall( + r'\b\d+\b', subprocess.check_output(f'{WORKER_DIR}/scripts/cpu_cores.sh', text=True) +)[0]; + PYTHON = os.getenv('PYTHON') or sys.executable; MESON = os.getenv('MESON') or f'{PIP_DIR}/bin/meson'; MESON_VERSION = os.getenv('MESON_VERSION') or '1.2.1'; @@ -55,6 +61,29 @@ # https://github.com/ninja-build/ninja/issues/2211 # https://github.com/ninja-build/ninja/issues/2212 NINJA_VERSION = os.getenv('NINJA_VERSION') or '1.10.2.4'; +NPM = os.getenv('NPM') or 'npm'; +LCOV = os.getenv('LCOV') or f'{WORKER_DIR}/deps/lcov/bin/lcov'; + +# Disable `*.pyc` files creation. +os.environ['PYTHONDONTWRITEBYTECODE'] = 'true'; + +# Instruct meson where to look for ninja binary. +if os.name == 'nt': + # Windows is, of course, special. + os.environ['NINJA'] = f'{PIP_DIR}/bin/ninja.exe'; +else: + os.environ['NINJA'] = f'{PIP_DIR}/bin/ninja'; + +# Instruct Python where to look for modules it needs, such that meson actually +# runs from installed location. +# NOTE: For some reason on Windows adding `:${PYTHONPATH}` breaks things. +PYTHONPATH = os.getenv('PYTHONPATH') or ''; +if os.name == 'nt': + os.environ['PYTHONPATH'] = PIP_DIR; +else: + os.environ['PYTHONPATH'] = f'{PIP_DIR}:{PYTHONPATH}'; + +# TODO: MESON_ARGS stuff (if not done above). # TODO: Remove. print('WORKER_DIR:', WORKER_DIR); @@ -62,6 +91,7 @@ print('INSTALL_DIR:', INSTALL_DIR); print('BUILD_DIR:', BUILD_DIR); print('PIP_DIR:', PIP_DIR); +print('NUM_CORES:', NUM_CORES); print('PYTHON:', PYTHON); print('MESON:', MESON); @@ -69,9 +99,8 @@ @task def meson_ninja(ctx): """ - Installs meson and ninja (also updates Python pip and setuptools packages) + Install meson and ninja (also update Python pip and setuptools packages) """ - if os.path.isdir(PIP_DIR): return; @@ -109,9 +138,8 @@ def meson_ninja(ctx): @task(pre=[meson_ninja]) def setup(ctx): """ - Runs meson setup + Run meson setup """ - # We try to call `--reconfigure` first as a workaround for this issue: # https://github.com/ninja-build/ninja/issues/1997 if MEDIASOUP_BUILDTYPE == 'Release': @@ -158,9 +186,8 @@ def setup(ctx): @task def clean(ctx): """ - Cleans the installation directory + Clean the installation directory """ - try: shutil.rmtree(INSTALL_DIR); except: @@ -170,9 +197,8 @@ def clean(ctx): @task def clean_build(ctx): """ - Cleans the build directory + Clean the build directory """ - try: shutil.rmtree(BUILD_DIR); except: @@ -182,9 +208,8 @@ def clean_build(ctx): @task def clean_pip(ctx): """ - Cleans the local pip directory + Clean the local pip directory """ - try: shutil.rmtree(PIP_DIR); except: @@ -194,9 +219,8 @@ def clean_pip(ctx): @task(pre=[meson_ninja]) def clean_subprojects(ctx): """ - Cleans meson subprojects + Clean meson subprojects """ - with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} subprojects purge --include-cache --confirm', @@ -207,9 +231,8 @@ def clean_subprojects(ctx): @task def clean_all(ctx): """ - Cleans meson subprojects and all installed/built artificats + Clean meson subprojects and all installed/built artificats """ - with ctx.cd(WORKER_DIR): try: ctx.run( @@ -226,9 +249,147 @@ def clean_all(ctx): @task(pre=[meson_ninja]) +def update_wrap_file(ctx, subproject): + """ + Update the wrap file of a subproject + """ + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} subprojects update --reset {subproject}', + echo=True + ); + + +@task(pre=[setup]) +def flatc(ctx): + """ + Compile FlatBuffers FBS files + """ + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} compile -C {BUILD_DIR} flatbuffers-generator', + echo=True + ); + + +@task(pre=[setup, flatc]) +def mediasoup_worker(ctx): + """ + Compile mediasoup-worker binary + """ + if os.getenv('MEDIASOUP_WORKER_BIN'): + printf('skipping mediasoup-worker compilation due to the existence of the MEDIASOUP_WORKER_BIN environment variable'); + return; + + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker', + echo=True + ); + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} compile -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker', + echo=True + ); + + +@task(pre=[setup, flatc]) +def libmediasoup_worker(ctx): + """ + Compile libmediasoup-worker library + """ + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} libmediasoup-worker', + echo=True + ); + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} compile -C {BUILD_DIR} --no-rebuild --tags libmediasoup-worker', + echo=True + ); + + +@task(pre=[setup, flatc]) +def xcode(ctx): + """ + Setup Xcode project + """ + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} setup --buildtype {MEDIASOUP_BUILDTYPE} --backend xcode {MEDIASOUP_OUT_DIR}/xcode', + echo=True + ); + + +@task +def lint(ctx): + """ + Lint source code + """ + with ctx.cd(WORKER_DIR): + ctx.run( + f'{NPM} run lint --prefix scripts/', + echo=True + ); + + +@task +def format(ctx): + """ + Format source code according to lint rules + """ + with ctx.cd(WORKER_DIR): + ctx.run( + f'{NPM} run format --prefix scripts/', + echo=True + ); + + +@task(pre=[setup, flatc]) +def test(ctx): + """ + Run worker tests + """ + ctx.run('echo $MEDIASOUP_LOCAL_DEV', echo=True); + + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-test', + echo=True + ); + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-test', + echo=True + ); + + MEDIASOUP_TEST_TAGS = os.getenv('MEDIASOUP_TEST_TAGS') or ''; + + # On Windows lcov doesn't work (at least not yet) and we need to add .exe to + # the binary path. + if os.name == 'nt': + with ctx.cd(WORKER_DIR): + ctx.run( + f'{BUILD_DIR}/mediasoup-worker-test.exe --invisibles --use-colour=yes {MEDIASOUP_TEST_TAGS}', + echo=True + ); + else: + ctx.run( + f'{LCOV} --directory {WORKER_DIR} --zerocounters', + echo=True + ); + with ctx.cd(WORKER_DIR): + ctx.run( + f'{BUILD_DIR}/mediasoup-worker-test --invisibles --use-colour=yes {MEDIASOUP_TEST_TAGS}', + echo=True + ); + + +@task(pre=[meson_ninja], default=True) def foo(ctx): """ - Foo things (until I figure out how to define a default task) + Foo things """ print('foo!'); From 6a31bdd7c663b3c8777bef4fa3f7449e0aa26f61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Thu, 23 Nov 2023 20:17:30 +0100 Subject: [PATCH 04/35] Let's see --- .github/workflows/mediasoup-node.yaml | 4 + .github/workflows/mediasoup-rust.yaml | 4 + .../workflows/mediasoup-worker-prebuild.yaml | 4 + .github/workflows/mediasoup-worker.yaml | 4 + TODO_INVOKE.md | 38 ++- npm-scripts.mjs | 74 +---- worker/Makefile | 22 +- worker/build.rs | 63 +--- worker/scripts/getmake.py | 28 -- worker/tasks.py | 290 +++++++++++++----- 10 files changed, 286 insertions(+), 245 deletions(-) delete mode 100644 worker/scripts/getmake.py diff --git a/.github/workflows/mediasoup-node.yaml b/.github/workflows/mediasoup-node.yaml index f22c707306..ae5ab5da51 100644 --- a/.github/workflows/mediasoup-node.yaml +++ b/.github/workflows/mediasoup-node.yaml @@ -47,6 +47,10 @@ jobs: restore-keys: | ${{ matrix.ci.os }}-node- + # TODO: Temporal until we decide how to deal with pip invoke requirement. + - name: pip install invoke + run: python3 -m pip install invoke + # NOTE: Add --force since otherwise `npm ci` fails in Node 16 because # @octokit/auth-token dev dependency requires Node >= 16. - name: npm ci diff --git a/.github/workflows/mediasoup-rust.yaml b/.github/workflows/mediasoup-rust.yaml index 0778129e84..f16947b1ef 100644 --- a/.github/workflows/mediasoup-rust.yaml +++ b/.github/workflows/mediasoup-rust.yaml @@ -38,6 +38,10 @@ jobs: ~/.cargo/git key: ${{ matrix.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} + # TODO: Temporal until we decide how to deal with pip invoke requirement. + - name: pip install invoke + run: python3 -m pip install invoke + - name: cargo fmt run: cargo fmt --all -- --check diff --git a/.github/workflows/mediasoup-worker-prebuild.yaml b/.github/workflows/mediasoup-worker-prebuild.yaml index 39392aae04..d9d34d91eb 100644 --- a/.github/workflows/mediasoup-worker-prebuild.yaml +++ b/.github/workflows/mediasoup-worker-prebuild.yaml @@ -50,6 +50,10 @@ jobs: with: node-version: ${{ matrix.node }} + # TODO: Temporal until we decide how to deal with pip invoke requirement. + - name: pip install invoke + run: python3 -m pip install invoke + # We need to install some NPM production deps for npm-scripts.mjs to work. - name: npm ci run: npm ci --ignore-scripts --omit=dev diff --git a/.github/workflows/mediasoup-worker.yaml b/.github/workflows/mediasoup-worker.yaml index f4e3bc16b6..717bf2df68 100644 --- a/.github/workflows/mediasoup-worker.yaml +++ b/.github/workflows/mediasoup-worker.yaml @@ -64,6 +64,10 @@ jobs: restore-keys: | ${{ matrix.build.os }}-node-${{matrix.build.cc}}- + # TODO: Temporal until we decide how to deal with pip invoke requirement. + - name: pip install invoke + run: python3 -m pip install invoke + # We need to install some NPM production deps for npm-scripts.mjs to work. - name: npm ci run: npm ci --ignore-scripts --omit=dev diff --git a/TODO_INVOKE.md b/TODO_INVOKE.md index 31c155f038..858b19063c 100644 --- a/TODO_INVOKE.md +++ b/TODO_INVOKE.md @@ -1,11 +1,43 @@ # TODO pip invoke -- I've renamed `MEDIASOUP_OUT_DIR` to `OUT_DIR`, so check if set/used somewhere else. +### pip `invoke` package is required + +This solution requires having the pip invoke package installed, however I think +we could just include it in the mediasoup repo in the worker/pip-invoke folder +and make npm-scripts.mjs invoke its executable in worker/pip-invoke/bin/invoke +(same for Rust). However that's hard since we also need to later override THONPATH +env for pip invoke module to be found. So... +Perhaps it's easier if we run worker/scripts/get-pip-invoke.sh during the +installation. That script installs pip invoke package in default pip location. +Or we can set PYTHONPATH env within npm-scripts.mjs (and Rust) and also in the +Makefile, so we don't remove it and instead make it behave as a proxy to invoke. +Let's see. + +In summary: +- `build.rs` needs to call `invoke` instead of `make`. Should we mandate `pip install invoke` when using mediasoup-rust? +- Same for `npm-scripts.mjs` which now should call `invoke` instead of `make`. +- Check if current `worker/scripts/get-pip-invoke.sh` script makes sense. + + +### Other TODO items + +- `MESON_ARGS` stuff. How to replicate this? What is this `subst` doing exactly? + ```Makefile + ifeq ($(OS),Windows_NT) + ifeq ($(MESON_ARGS),"") + MESON_ARGS = $(subst $\",,"--vsenv") + endif + endif + ``` - `build.rs`: Remvoe the `MEDIASOUP_OUT_DIR` env stuff. - Remove `getmake.py` and related stuff. -- When running meson commands we are loosing colored output, perhaps some invoke `context.run()` option is needed. Reported here: https://github.com/pyinvoke/invoke/discussions/976 +- When running meson commands we are loosing colored output, perhaps some invoke `context.run()` option is needed. Reported here: https://github.com/pyinvoke/invoke/discussions/976. Solved using `pty=True` in `ctx.run()` but I need to check the implications. So I'm adding `pty=True` everywehere. Let's see. -- Every time I run `invoke test` or `make test` the whole worker is built from scratch :( This may happen because we run `flatc` task anyway in `tasks.py` (maybe a TODO), but why when using `make test`??? +- Do we still need this in `build.rs`? + ```rust + // Force forward slashes on Windows too so that is plays well with our dumb `Makefile`. + let mediasoup_out_dir = format!("{}/out", out_dir.replace('\\', "/")); + ``` diff --git a/npm-scripts.mjs b/npm-scripts.mjs index 2de2298c8a..c4feea35d1 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -2,15 +2,14 @@ import process from 'node:process'; import os from 'node:os'; import fs from 'node:fs'; import path from 'node:path'; -import { execSync, spawnSync } from 'node:child_process'; +import { execSync } from 'node:child_process'; import fetch from 'node-fetch'; import tar from 'tar'; const PKG = JSON.parse(fs.readFileSync('./package.json').toString()); -const IS_FREEBSD = os.platform() === 'freebsd'; const IS_WINDOWS = os.platform() === 'win32'; const MAYOR_VERSION = PKG.version.split('.')[0]; -const MAKE = process.env.MAKE || (IS_FREEBSD ? 'gmake' : 'make'); +const INVOKE = process.env.INVOKE || 'invoke'; const FLATBUFFERS_VERSION = '23.3.3'; const WORKER_RELEASE_DIR = 'worker/out/Release'; const WORKER_RELEASE_BIN = IS_WINDOWS ? 'mediasoup-worker.exe' : 'mediasoup-worker'; @@ -142,7 +141,7 @@ async function run() case 'format:worker': { - executeCmd(`${MAKE} format -C worker`); + executeCmd(`${INVOKE} -r worker format`); break; } @@ -311,22 +310,7 @@ function buildWorker() { logInfo('buildWorker()'); - if (IS_WINDOWS) - { - if (!fs.existsSync('worker/out/msys/bin/make.exe')) - { - installMsysMake(); - } - - const msysPath = `${process.cwd()}\\worker\\out\\msys\\bin`; - - if (!process.env.PATH.includes(msysPath)) - { - process.env.PATH = `${msysPath};${process.env.PATH}`; - } - } - - executeCmd(`${MAKE} -C worker`); + executeCmd(`${INVOKE} -r worker`); } function cleanWorkerArtifacts() @@ -334,11 +318,11 @@ function cleanWorkerArtifacts() logInfo('cleanWorkerArtifacts()'); // Clean build artifacts except `mediasoup-worker`. - executeCmd(`${MAKE} clean-build -C worker`); + executeCmd(`${INVOKE} -r worker clean-build`); // Clean downloaded dependencies. - executeCmd(`${MAKE} clean-subprojects -C worker`); + executeCmd(`${INVOKE} -r worker clean-subprojects`); // Clean PIP/Meson/Ninja. - executeCmd(`${MAKE} clean-pip -C worker`); + executeCmd(`${INVOKE} -r worker clean-pip`); if (IS_WINDOWS) { @@ -360,7 +344,7 @@ function lintWorker() { logInfo('lintWorker()'); - executeCmd(`${MAKE} lint -C worker`); + executeCmd(`${INVOKE} -r worker lint`); } function flatcNode() @@ -368,7 +352,7 @@ function flatcNode() logInfo('flatcNode()'); // Build flatc if needed. - executeCmd(`${MAKE} -C worker flatc`); + executeCmd(`${INVOKE} -r worker flatc`); const buildType = process.env.MEDIASOUP_BUILDTYPE || 'Release'; const extension = IS_WINDOWS ? '.exe' : ''; @@ -393,7 +377,7 @@ function flatcWorker() { logInfo('flatcWorker()'); - executeCmd(`${MAKE} -C worker flatc`); + executeCmd(`${INVOKE} -r worker flatc`); } function testNode() @@ -414,7 +398,7 @@ function testWorker() { logInfo('testWorker()'); - executeCmd(`${MAKE} test -C worker`); + executeCmd(`${INVOKE} -r worker test`); } function installNodeDeps() @@ -441,42 +425,6 @@ function checkRelease() testWorker(); } -function installMsysMake() -{ - logInfo('installMsysMake()'); - - let pythonPath; - - // If PYTHON environment variable is given, use it. - if (process.env.PYTHON) - { - pythonPath = process.env.PYTHON; - } - // Otherwise ensure python3.exe is available in the PATH. - else - { - let res = spawnSync('where', [ 'python3.exe' ]); - - if (res.status !== 0) - { - res = spawnSync('where', [ 'python.exe' ]); - - if (res.status !== 0) - { - logError('`installMsysMake() | cannot find Python executable'); - - exitWithError(); - } - } - - pythonPath = String(res.stdout).trim(); - } - - const dir = path.resolve('worker/out/msys'); - - executeCmd(`${pythonPath} worker\\scripts\\getmake.py --dir="${dir}"`); -} - function ensureDir(dir) { logInfo(`ensureDir() [dir:${dir}]`); diff --git a/worker/Makefile b/worker/Makefile index 1bf97aebf6..b7e0faed02 100644 --- a/worker/Makefile +++ b/worker/Makefile @@ -16,8 +16,8 @@ MEDIASOUP_BUILDTYPE ?= Release LCOV = $(WORKER_DIR)/deps/lcov/bin/lcov DOCKER ?= docker PIP_DIR = $(MEDIASOUP_OUT_DIR)/pip -INSTALL_DIR ?= $(MEDIASOUP_OUT_DIR)/$(MEDIASOUP_BUILDTYPE) -BUILD_DIR ?= $(INSTALL_DIR)/build +MEDIASOUP_INSTALL_DIR ?= $(MEDIASOUP_OUT_DIR)/$(MEDIASOUP_BUILDTYPE) +BUILD_DIR ?= $(MEDIASOUP_INSTALL_DIR)/build MESON ?= $(PIP_DIR)/bin/meson MESON_VERSION ?= 1.2.1 # `MESON_ARGS` can be used to provide extra configuration parameters to Meson, @@ -113,7 +113,7 @@ setup: meson-ninja # https://github.com/ninja-build/ninja/issues/1997 ifeq ($(MEDIASOUP_BUILDTYPE),Release) $(MESON) setup \ - --prefix $(INSTALL_DIR) \ + --prefix $(MEDIASOUP_INSTALL_DIR) \ --bindir "" \ --libdir "" \ --buildtype release \ @@ -124,7 +124,7 @@ ifeq ($(MEDIASOUP_BUILDTYPE),Release) $(MESON_ARGS) \ $(BUILD_DIR) || \ $(MESON) setup \ - --prefix $(INSTALL_DIR) \ + --prefix $(MEDIASOUP_INSTALL_DIR) \ --bindir "" \ --libdir "" \ --buildtype release \ @@ -136,7 +136,7 @@ ifeq ($(MEDIASOUP_BUILDTYPE),Release) else ifeq ($(MEDIASOUP_BUILDTYPE),Debug) $(MESON) setup \ - --prefix $(INSTALL_DIR) \ + --prefix $(MEDIASOUP_INSTALL_DIR) \ --bindir "" \ --libdir "" \ --buildtype debug \ @@ -146,7 +146,7 @@ ifeq ($(MEDIASOUP_BUILDTYPE),Debug) $(MESON_ARGS) \ $(BUILD_DIR) || \ $(MESON) setup \ - --prefix $(INSTALL_DIR) \ + --prefix $(MEDIASOUP_INSTALL_DIR) \ --bindir "" \ --libdir "" \ --buildtype debug \ @@ -156,7 +156,7 @@ ifeq ($(MEDIASOUP_BUILDTYPE),Debug) $(BUILD_DIR) else $(MESON) setup \ - --prefix $(INSTALL_DIR) \ + --prefix $(MEDIASOUP_INSTALL_DIR) \ --bindir "" \ --libdir "" \ --buildtype $(MEDIASOUP_BUILDTYPE) \ @@ -167,7 +167,7 @@ else $(MESON_ARGS) \ $(BUILD_DIR) || \ $(MESON) setup \ - --prefix $(INSTALL_DIR) \ + --prefix $(MEDIASOUP_INSTALL_DIR) \ --bindir "" \ --libdir "" \ --buildtype $(MEDIASOUP_BUILDTYPE) \ @@ -180,7 +180,7 @@ endif endif clean: - $(RM) -rf $(INSTALL_DIR) + $(RM) -rf $(MEDIASOUP_INSTALL_DIR) clean-build: $(RM) -rf $(BUILD_DIR) @@ -269,7 +269,7 @@ docker-run: --name=mediasoupDocker -it --rm \ --privileged \ --cap-add SYS_PTRACE \ - -v $(shell pwd)/../:/mediasoup \ + -v $(WORKER_DIR)/../:/mediasoup \ mediasoup/docker:latest docker-alpine: @@ -284,5 +284,5 @@ docker-alpine-run: --name=mediasoupDockerAlpine -it --rm \ --privileged \ --cap-add SYS_PTRACE \ - -v $(shell pwd)/../:/mediasoup \ + -v $(WORKER_DIR)/../:/mediasoup \ mediasoup/docker-alpine:latest diff --git a/worker/build.rs b/worker/build.rs index 8bf0b836fe..32ca69de79 100644 --- a/worker/build.rs +++ b/worker/build.rs @@ -45,15 +45,9 @@ fn main() { ) .expect("Failed to write generated Rust flatbuffers into fbs.rs"); - // Force forward slashes on Windows too so that is plays well with our dumb `Makefile`. + // Force forward slashes on Windows too so that is plays well with our tasks.py. let mediasoup_out_dir = format!("{}/out", out_dir.replace('\\', "/")); - // Store original PATH so we make `make clean-all` use it. This is because, in Windows, - // we may need to fetch `make` and we store it in out/msys and then we add out/msys/bin - // to the PATH, and that folder may contain rm.exe, so `make clean-all` would use - // that rm.exe and delete itself and make the task fail. - let original_path = env::var("PATH").unwrap(); - // Add C++ std lib #[cfg(target_os = "linux")] { @@ -110,62 +104,14 @@ fn main() { println!("cargo:rustc-link-lib=dylib=c++"); println!("cargo:rustc-link-lib=dylib=c++abi"); } - #[cfg(target_os = "windows")] - { - if !std::path::Path::new("worker/out/msys/bin/make.exe").exists() { - let python = if let Ok(python) = env::var("PYTHON") { - python - } else if Command::new("where") - .arg("python3") - .status() - .expect("Failed to start") - .success() - { - "python3".to_string() - } else { - "python".to_string() - }; - - let dir = format!("{}/msys", mediasoup_out_dir.replace('\\', "/")); - - if !Command::new(python) - .arg("scripts\\getmake.py") - .arg("--dir") - .arg(dir.clone()) - .status() - .expect("Failed to start") - .success() - { - panic!("Failed to install MSYS/make") - } - - env::set_var( - "PATH", - format!("{}\\bin;{}", dir, env::var("PATH").unwrap()), - ); - } - - env::set_var( - "PATH", - format!( - "{}\\worker\\out\\msys\\bin;{}", - env::current_dir() - .unwrap() - .into_os_string() - .into_string() - .unwrap(), - env::var("PATH").unwrap() - ), - ); - } // Build - if !Command::new("make") + if !Command::new("invoke") .arg("libmediasoup-worker") .env("MEDIASOUP_OUT_DIR", &mediasoup_out_dir) .env("MEDIASOUP_BUILDTYPE", build_type) // Force forward slashes on Windows too, otherwise Meson thinks path is not absolute 🤷 - .env("INSTALL_DIR", &out_dir.replace('\\', "/")) + .env("MEDIASOUP_INSTALL_DIR", &out_dir.replace('\\', "/")) .spawn() .expect("Failed to start") .wait() @@ -205,9 +151,8 @@ fn main() { if env::var("KEEP_BUILD_ARTIFACTS") != Ok("1".to_string()) { // Clean - if !Command::new("make") + if !Command::new("invoke") .arg("clean-all") - .env("PATH", original_path) .env("MEDIASOUP_OUT_DIR", &mediasoup_out_dir) .spawn() .expect("Failed to start") diff --git a/worker/scripts/getmake.py b/worker/scripts/getmake.py deleted file mode 100644 index 50ec9ea7f1..0000000000 --- a/worker/scripts/getmake.py +++ /dev/null @@ -1,28 +0,0 @@ -import argparse, io, hashlib, tarfile, urllib.request - -argParser = argparse.ArgumentParser() - -argParser.add_argument( - '--dir', - type=str, - required=True, - help='absolute path of the directoy in which fetched content will be placed' -) - -args = argParser.parse_args() - -def get(url, digest): - data = urllib.request.urlopen(url).read() - assert hashlib.sha256(data).hexdigest() == digest - tar = tarfile.open(fileobj=io.BytesIO(data)) - tar.extractall(args.dir) - tar.close() - -get('https://sourceforge.net/projects/mingw/files/MSYS/Base/msys-core/msys-1.0.19-1/msysCORE-1.0.19-1-msys-1.0.19-bin.tar.xz/download', '8c4157d739a460f85563bc4451e9f1bbd42b13c4f63770d43b9f45a781f07858') -get('https://sourceforge.net/projects/mingw/files/MSYS/Base/libiconv/libiconv-1.14-1/libiconv-1.14-1-msys-1.0.17-dll-2.tar.lzma/download', '196921e8c232259c8e6a6852b9ee8d9ab2d29a91419f0c8dc27ba6f034231683') -get('https://sourceforge.net/projects/mingw/files/MSYS/Base/gettext/gettext-0.18.1.1-1/libintl-0.18.1.1-1-msys-1.0.17-dll-8.tar.lzma/download', '29db8c969661c511fbe2a341ab25c993c5f9c555842a75d6ddbcfa70dec16910') -get('https://sourceforge.net/projects/mingw/files/MSYS/Base/coreutils/coreutils-5.97-3/coreutils-5.97-3-msys-1.0.13-bin.tar.lzma/download', 'f8c7990416ea16a74ac336dcfe0f596bc46b8724b2d58cf8a3509414220b2366') -get('https://sourceforge.net/projects/mingw/files/MSYS/Base/regex/regex-1.20090805-2/libregex-1.20090805-2-msys-1.0.13-dll-1.tar.lzma/download', '85dd8c1e27a90675c5f867be57ba7ae2bb55dde8cd2d19f284c896be134bd3d1') -get('https://sourceforge.net/projects/mingw/files/MSYS/Base/termcap/termcap-0.20050421_1-2/libtermcap-0.20050421_1-2-msys-1.0.13-dll-0.tar.lzma/download', '62b58fe0880f0972fcc84a819265989b02439c1c5185870227bd25f870f7adb6') -get('https://sourceforge.net/projects/mingw/files/MSYS/Base/bash/bash-3.1.23-1/bash-3.1.23-1-msys-1.0.18-bin.tar.xz/download', '38da5419969ab883058a96322bb0f51434dd4e9f71de09cd4f75b96750944533') -get('https://sourceforge.net/projects/mingw/files/MSYS/Base/make/make-3.81-3/make-3.81-3-msys-1.0.13-bin.tar.lzma/download', '847f0cbbf07135801c8e67bf692d29b1821e816ad828753c997fa869a9b89988') diff --git a/worker/tasks.py b/worker/tasks.py index 2996955e8d..025c2ddcaa 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -1,20 +1,11 @@ -# This is a tasks.py file for the pip invoke package: https://docs.pyinvoke.org/. # -# It's a replacement of our Makefile with same tasks. Obviously it requires -# having the pip invoke package installed, however I think we could just -# include it in the mediasoup repo in the worker/pip-invoke folder and make -# npm-scripts.mjs invoke its executable in worker/pip-invoke/bin/invoke (same -# for Rust). However that's hard since we also need to later override PYTHONPATH -# env for pip invoke module to be found. So... +# This is a tasks.py file for the pip invoke package: https://docs.pyinvoke.org/. # -# Perhaps it's easier if we run worker/scripts/get-pip-invoke.sh during the -# installation. That script installs pip invoke package in default pip location. +# It's a replacement of our Makefile with same tasks. # -# Or we can set PYTHONPATH env within npm-scripts.mjs (and Rust) and also in the -# Makefile, so we don't remove it and instead make it behave as a proxy to invoke. +# Usage: +# invoke --list # -# Let's see. - import sys; import os; @@ -26,19 +17,17 @@ MEDIASOUP_BUILDTYPE = os.getenv('MEDIASOUP_BUILDTYPE') or 'Release'; - WORKER_DIR = os.path.dirname(os.path.abspath( inspect.getframeinfo(inspect.currentframe()).filename )); -OUT_DIR = os.getenv('OUT_DIR') or f'{WORKER_DIR}/out'; -INSTALL_DIR = os.getenv('INSTALL_DIR') or f'{OUT_DIR}/{MEDIASOUP_BUILDTYPE}'; -BUILD_DIR = os.getenv('BUILD_DIR') or f'{INSTALL_DIR}/build'; -PIP_DIR = f'{OUT_DIR}/pip'; - +# NOTE: MEDIASOUP_OUT_DIR is overrided by build.rs. +MEDIASOUP_OUT_DIR = os.getenv('MEDIASOUP_OUT_DIR') or f'{WORKER_DIR}/out'; +MEDIASOUP_INSTALL_DIR = os.getenv('MEDIASOUP_INSTALL_DIR') or f'{MEDIASOUP_OUT_DIR}/{MEDIASOUP_BUILDTYPE}'; +BUILD_DIR = os.getenv('BUILD_DIR') or f'{MEDIASOUP_INSTALL_DIR}/build'; +PIP_DIR = f'{MEDIASOUP_OUT_DIR}/pip'; NUM_CORES = re.findall( r'\b\d+\b', subprocess.check_output(f'{WORKER_DIR}/scripts/cpu_cores.sh', text=True) )[0]; - PYTHON = os.getenv('PYTHON') or sys.executable; MESON = os.getenv('MESON') or f'{PIP_DIR}/bin/meson'; MESON_VERSION = os.getenv('MESON_VERSION') or '1.2.1'; @@ -48,14 +37,9 @@ # worker with tracing and enabled. # NOTE: On Windows make sure to add `--vsenv` or have MSVS environment already # active if you override this parameter. -# TODO: Replicate this: -# # Activate VS environment on Windows by default. -# ifeq ($(OS),Windows_NT) -# ifeq ($(MESON_ARGS),"") -# MESON_ARGS = $(subst $\",,"--vsenv") -# endif -# endif MESON_ARGS = ''; +if os.name == 'nt' and not os.getenv('MESON_ARGS'): + MESON_ARGS = '--vsenv'; # Let's use a specific version of ninja to avoid buggy version 1.11.1: # https://mediasoup.discourse.group/t/partly-solved-could-not-detect-ninja-v1-8-2-or-newer/ # https://github.com/ninja-build/ninja/issues/2211 @@ -63,6 +47,7 @@ NINJA_VERSION = os.getenv('NINJA_VERSION') or '1.10.2.4'; NPM = os.getenv('NPM') or 'npm'; LCOV = os.getenv('LCOV') or f'{WORKER_DIR}/deps/lcov/bin/lcov'; +DOCKER = os.getenv('DOCKER') or 'docker'; # Disable `*.pyc` files creation. os.environ['PYTHONDONTWRITEBYTECODE'] = 'true'; @@ -76,25 +61,13 @@ # Instruct Python where to look for modules it needs, such that meson actually # runs from installed location. -# NOTE: For some reason on Windows adding `:${PYTHONPATH}` breaks things. -PYTHONPATH = os.getenv('PYTHONPATH') or ''; +# NOTE: For some reason on Windows adding `:{PYTHONPATH}` breaks things. if os.name == 'nt': os.environ['PYTHONPATH'] = PIP_DIR; else: + PYTHONPATH = os.getenv('PYTHONPATH') or ''; os.environ['PYTHONPATH'] = f'{PIP_DIR}:{PYTHONPATH}'; -# TODO: MESON_ARGS stuff (if not done above). - -# TODO: Remove. -print('WORKER_DIR:', WORKER_DIR); -print('OUT_DIR:', OUT_DIR); -print('INSTALL_DIR:', INSTALL_DIR); -print('BUILD_DIR:', BUILD_DIR); -print('PIP_DIR:', PIP_DIR); -print('NUM_CORES:', NUM_CORES); -print('PYTHON:', PYTHON); -print('MESON:', MESON); - @task def meson_ninja(ctx): @@ -117,7 +90,8 @@ def meson_ninja(ctx): except: ctx.run( f'{PYTHON} -m pip install --target={PIP_DIR} pip setuptools', - echo=True + echo=True, + pty=True ); # Workaround for NixOS and Guix that don't work with pre-built binaries, see: @@ -131,7 +105,8 @@ def meson_ninja(ctx): # depend on system-wide installation. ctx.run( f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} {PIP_BUILD_BINARIES} meson=={MESON_VERSION} ninja=={NINJA_VERSION}', - echo=True + echo=True, + pty=True ); @@ -146,40 +121,46 @@ def setup(ctx): try: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', - echo=True + f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', + echo=True, + pty=True ); except: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', - echo=True + f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', + echo=True, + pty=True ); elif MEDIASOUP_BUILDTYPE == 'Debug': try: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', - echo=True + f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', + echo=True, + pty=True ); except: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', - echo=True + f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', + echo=True, + pty=True ); else: try: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', - echo=True + f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', + echo=True, + pty=True ); except: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', - echo=True + f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', + echo=True, + pty=True ); @@ -189,7 +170,7 @@ def clean(ctx): Clean the installation directory """ try: - shutil.rmtree(INSTALL_DIR); + shutil.rmtree(MEDIASOUP_INSTALL_DIR); except: pass; @@ -224,7 +205,8 @@ def clean_subprojects(ctx): with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} subprojects purge --include-cache --confirm', - echo=True + echo=True, + pty=True ); @@ -237,13 +219,14 @@ def clean_all(ctx): try: ctx.run( f'{MESON} subprojects purge --include-cache --confirm', - echo=True + echo=True, + pty=True ); except: pass; try: - shutil.rmtree(OUT_DIR); + shutil.rmtree(MEDIASOUP_OUT_DIR); except: pass; @@ -256,7 +239,8 @@ def update_wrap_file(ctx, subproject): with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} subprojects update --reset {subproject}', - echo=True + echo=True, + pty=True ); @@ -268,11 +252,12 @@ def flatc(ctx): with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} compile -C {BUILD_DIR} flatbuffers-generator', - echo=True + echo=True, + pty=True ); -@task(pre=[setup, flatc]) +@task(pre=[setup, flatc], default=True) def mediasoup_worker(ctx): """ Compile mediasoup-worker binary @@ -284,12 +269,14 @@ def mediasoup_worker(ctx): with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker', - echo=True + echo=True, + pty=True ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} compile -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker', - echo=True + echo=True, + pty=True ); @@ -301,12 +288,14 @@ def libmediasoup_worker(ctx): with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} libmediasoup-worker', - echo=True + echo=True, + pty=True ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} compile -C {BUILD_DIR} --no-rebuild --tags libmediasoup-worker', - echo=True + echo=True, + pty=True ); @@ -318,7 +307,8 @@ def xcode(ctx): with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} setup --buildtype {MEDIASOUP_BUILDTYPE} --backend xcode {MEDIASOUP_OUT_DIR}/xcode', - echo=True + echo=True, + pty=True ); @@ -330,7 +320,8 @@ def lint(ctx): with ctx.cd(WORKER_DIR): ctx.run( f'{NPM} run lint --prefix scripts/', - echo=True + echo=True, + pty=True ); @@ -342,7 +333,8 @@ def format(ctx): with ctx.cd(WORKER_DIR): ctx.run( f'{NPM} run format --prefix scripts/', - echo=True + echo=True, + pty=True ); @@ -351,17 +343,17 @@ def test(ctx): """ Run worker tests """ - ctx.run('echo $MEDIASOUP_LOCAL_DEV', echo=True); - with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-test', - echo=True + echo=True, + pty=True ); with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-test', - echo=True + f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-test', + echo=True, + pty=True ); MEDIASOUP_TEST_TAGS = os.getenv('MEDIASOUP_TEST_TAGS') or ''; @@ -372,25 +364,161 @@ def test(ctx): with ctx.cd(WORKER_DIR): ctx.run( f'{BUILD_DIR}/mediasoup-worker-test.exe --invisibles --use-colour=yes {MEDIASOUP_TEST_TAGS}', - echo=True + echo=True, + pty=True ); else: ctx.run( f'{LCOV} --directory {WORKER_DIR} --zerocounters', - echo=True + echo=True, + pty=True ); with ctx.cd(WORKER_DIR): ctx.run( f'{BUILD_DIR}/mediasoup-worker-test --invisibles --use-colour=yes {MEDIASOUP_TEST_TAGS}', - echo=True + echo=True, + pty=True + ); + + +@task(pre=[setup, flatc]) +def test_asan(ctx): + """ + Run worker test with Address Sanitizer + """ + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-test-asan', + echo=True, + pty=True + ); + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-test-asan', + echo=True, + pty=True + ); + + MEDIASOUP_TEST_TAGS = os.getenv('MEDIASOUP_TEST_TAGS') or ''; + + with ctx.cd(WORKER_DIR): + ctx.run( + f'ASAN_OPTIONS=detect_leaks=1 {BUILD_DIR}/mediasoup-worker-test-asan --invisibles --use-colour=yes {MEDIASOUP_TEST_TAGS}', + echo=True, + pty=True + ); + + +@task +def tidy(ctx): + """ + Perform C++ checks with clang-tidy + """ + MEDIASOUP_TIDY_CHECKS = os.getenv('MEDIASOUP_TIDY_CHECKS') or ''; + + with ctx.cd(WORKER_DIR): + ctx.run( + f'{PYTHON} ./scripts/clang-tidy.py -clang-tidy-binary=./scripts/node_modules/.bin/clang-tidy -clang-apply-replacements-binary=./scripts/node_modules/.bin/clang-apply-replacements -header-filter="(Channel/**/*.hpp|DepLibSRTP.hpp|DepLibUV.hpp|DepLibWebRTC.hpp|DepOpenSSL.hpp|DepUsrSCTP.hpp|LogLevel.hpp|Logger.hpp|MediaSoupError.hpp|RTC/**/*.hpp|Settings.hpp|Utils.hpp|Worker.hpp|common.hpp|handles/**/*.hpp)" -p={BUILD_DIR} -j={NUM_CORES} -checks={MEDIASOUP_TIDY_CHECKS} -quiet', + echo=True, + pty=True + ); + + +@task(pre=[setup, flatc]) +def fuzzer(ctx): + """ + Build the mediasoup-worker-fuzzer binary (which uses libFuzzer) + """ + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-fuzzer', + echo=True, + pty=True + ); + with ctx.cd(WORKER_DIR): + ctx.run( + f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-fuzzer', + echo=True, + pty=True + ); + + +@task +def fuzzer(ctx): + """ + Run all fuzzer cases + """ + with ctx.cd(WORKER_DIR): + ctx.run( + f'LSAN_OPTIONS=verbosity=1:log_threads=1 {BUILD_DIR}/mediasoup-worker-fuzzer -artifact_prefix=fuzzer/reports/ -max_len=1400 fuzzer/new-corpus deps/webrtc-fuzzer-corpora/corpora/stun-corpus deps/webrtc-fuzzer-corpora/corpora/rtp-corpus deps/webrtc-fuzzer-corpora/corpora/rtcp-corpus', + echo=True, + pty=True + ); + + +@task +def docker(ctx): + """ + Build a Linux Ubuntu Docker image with fuzzer capable clang++ + """ + if os.getenv('DOCKER_NO_CACHE') == 'true': + with ctx.cd(WORKER_DIR): + ctx.run( + f'{DOCKER} build -f Dockerfile --no-cache --tag mediasoup/docker:latest .', + echo=True, + pty=True ); + else: + with ctx.cd(WORKER_DIR): + ctx.run( + f'{DOCKER} build -f Dockerfile --tag mediasoup/docker:latest .', + echo=True, + pty=True + ); + + +@task +def docker_run(ctx): + """ + Run a container of the Ubuntu Docker image created in the docker task + """ + with ctx.cd(WORKER_DIR): + ctx.run( + f'{DOCKER} run --name=mediasoupDocker -it --rm --privileged --cap-add SYS_PTRACE -v {WORKER_DIR}/../:/mediasoup mediasoup/docker:latest', + echo=True, + pty=True # NOTE: Needed to enter the terminal of the Docker image. + ); -@task(pre=[meson_ninja], default=True) -def foo(ctx): +@task +def docker_alpine(ctx): """ - Foo things + Build a Linux Alpine Docker image """ + if os.getenv('DOCKER_NO_CACHE') == 'true': + with ctx.cd(WORKER_DIR): + ctx.run( + f'{DOCKER} build -f Dockerfile.alpine --no-cache --tag mediasoup/docker-alpine:latest .', + echo=True, + pty=True + ); + else: + with ctx.cd(WORKER_DIR): + ctx.run( + f'{DOCKER} build -f Dockerfile.alpine --tag mediasoup/docker-alpine:latest .', + echo=True, + pty=True + ); - print('foo!'); +@task +def docker_alpine_run(ctx): + """ + Run a container of the Alpine Docker image created in the docker_alpine task + """ + with ctx.cd(WORKER_DIR): + ctx.run( + f'{DOCKER} run --name=mediasoupDockerAlpine -it --rm --privileged --cap-add SYS_PTRACE -v {WORKER_DIR}/../:/mediasoup mediasoup/docker-alpine:latest', + echo=True, + pty=True # NOTE: Needed to enter the terminal of the Docker image. + ); From 640edb5b0aeafbd0e323a7b2041e760e7060acc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Thu, 23 Nov 2023 20:23:07 +0100 Subject: [PATCH 05/35] fixes --- TODO_INVOKE.md | 2 ++ worker/tasks.py | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/TODO_INVOKE.md b/TODO_INVOKE.md index 858b19063c..2a37582ca7 100644 --- a/TODO_INVOKE.md +++ b/TODO_INVOKE.md @@ -41,3 +41,5 @@ In summary: // Force forward slashes on Windows too so that is plays well with our dumb `Makefile`. let mediasoup_out_dir = format!("{}/out", out_dir.replace('\\', "/")); ``` + +- "Code QL Analyze (cpp)" CI task fails because "Command failed: invoke -r worker" (as expected): https://github.com/versatica/mediasoup/actions/runs/6973590508/job/18977838894?pr=1239 diff --git a/worker/tasks.py b/worker/tasks.py index 025c2ddcaa..fa71b8eb06 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -274,7 +274,7 @@ def mediasoup_worker(ctx): ); with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} compile -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker', + f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker', echo=True, pty=True ); @@ -293,7 +293,7 @@ def libmediasoup_worker(ctx): ); with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} compile -C {BUILD_DIR} --no-rebuild --tags libmediasoup-worker', + f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags libmediasoup-worker', echo=True, pty=True ); From 1a3914bb3c7749c444037df3310c19d7f8004b09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 00:53:36 +0100 Subject: [PATCH 06/35] Don't need cpu_cores.sh anymore --- TODO_INVOKE.md | 4 ++++ worker/tasks.py | 26 +++++++++++++++++++++----- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/TODO_INVOKE.md b/TODO_INVOKE.md index 2a37582ca7..897be87421 100644 --- a/TODO_INVOKE.md +++ b/TODO_INVOKE.md @@ -43,3 +43,7 @@ In summary: ``` - "Code QL Analyze (cpp)" CI task fails because "Command failed: invoke -r worker" (as expected): https://github.com/versatica/mediasoup/actions/runs/6973590508/job/18977838894?pr=1239 + +- Once we have `invoke` working, let's remove the no longer used `cpu_cores.sh` script. + +- And of course, remove `Makefile`. diff --git a/worker/tasks.py b/worker/tasks.py index fa71b8eb06..1a0b221a55 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -11,8 +11,6 @@ import os; import inspect; import shutil; -import subprocess; -import re; from invoke import task; @@ -25,9 +23,11 @@ MEDIASOUP_INSTALL_DIR = os.getenv('MEDIASOUP_INSTALL_DIR') or f'{MEDIASOUP_OUT_DIR}/{MEDIASOUP_BUILDTYPE}'; BUILD_DIR = os.getenv('BUILD_DIR') or f'{MEDIASOUP_INSTALL_DIR}/build'; PIP_DIR = f'{MEDIASOUP_OUT_DIR}/pip'; -NUM_CORES = re.findall( - r'\b\d+\b', subprocess.check_output(f'{WORKER_DIR}/scripts/cpu_cores.sh', text=True) -)[0]; +# If available (only on some *nix systems), os.sched_getaffinity(0) gets set of +# CPUs the calling thread is restricted to. Instead, os.cpu_count() returns the +# total number of CPUs in a system (it doesn't take into account how many of them +# the calling thread can use). +NUM_CORES = len(os.sched_getaffinity(0)) if hasattr(os, 'sched_getaffinity') else os.cpu_count(); PYTHON = os.getenv('PYTHON') or sys.executable; MESON = os.getenv('MESON') or f'{PIP_DIR}/bin/meson'; MESON_VERSION = os.getenv('MESON_VERSION') or '1.2.1'; @@ -69,6 +69,22 @@ os.environ['PYTHONPATH'] = f'{PIP_DIR}:{PYTHONPATH}'; +# Utils. + +def get_num_cpus(): + # If available (only on some *nix systems), os.sched_getaffinity(0) gets + # accurate set of CPUs the calling thread is restricted to. + # + if hasattr(os, 'sched_getaffinity'): + return len(os.sched_getaffinity(0)); + # Instead, os.cpu_count() returns the count of the total CPUs in a system + # (it doesn't take into account how many of them the calling thread can use). + else: + return os.cpu_info(); + + +# Tasks. + @task def meson_ninja(ctx): """ From 26c4faf2e3a12755a0872e34a3c30cbaf33453d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 01:05:04 +0100 Subject: [PATCH 07/35] improve tasks.py by manually running pylint on it --- worker/tasks.py | 47 +++++++++++++---------------------------------- 1 file changed, 13 insertions(+), 34 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 1a0b221a55..7c4a6d17d6 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -37,9 +37,7 @@ # worker with tracing and enabled. # NOTE: On Windows make sure to add `--vsenv` or have MSVS environment already # active if you override this parameter. -MESON_ARGS = ''; -if os.name == 'nt' and not os.getenv('MESON_ARGS'): - MESON_ARGS = '--vsenv'; +MESON_ARGS = '--vsenv' if os.name == 'nt' and not os.getenv('MESON_ARGS') else ''; # Let's use a specific version of ninja to avoid buggy version 1.11.1: # https://mediasoup.discourse.group/t/partly-solved-could-not-detect-ninja-v1-8-2-or-newer/ # https://github.com/ninja-build/ninja/issues/2211 @@ -69,22 +67,6 @@ os.environ['PYTHONPATH'] = f'{PIP_DIR}:{PYTHONPATH}'; -# Utils. - -def get_num_cpus(): - # If available (only on some *nix systems), os.sched_getaffinity(0) gets - # accurate set of CPUs the calling thread is restricted to. - # - if hasattr(os, 'sched_getaffinity'): - return len(os.sched_getaffinity(0)); - # Instead, os.cpu_count() returns the count of the total CPUs in a system - # (it doesn't take into account how many of them the calling thread can use). - else: - return os.cpu_info(); - - -# Tasks. - @task def meson_ninja(ctx): """ @@ -112,15 +94,12 @@ def meson_ninja(ctx): # Workaround for NixOS and Guix that don't work with pre-built binaries, see: # https://github.com/NixOS/nixpkgs/issues/142383. - PIP_BUILD_BINARIES = ''; - - if os.path.isfile('/etc/NIXOS') or os.path.isdir('/etc/guix'): - PIP_BUILD_BINARIES = '--no-binary :all:'; + pip_build_binaries = '--no-binary :all:' if os.path.isfile('/etc/NIXOS') or os.path.isdir('/etc/guix') else ''; # Install meson and ninja using pip into our custom location, so we don't # depend on system-wide installation. ctx.run( - f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} {PIP_BUILD_BINARIES} meson=={MESON_VERSION} ninja=={NINJA_VERSION}', + f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} {pip_build_binaries} meson=={MESON_VERSION} ninja=={NINJA_VERSION}', echo=True, pty=True ); @@ -131,7 +110,7 @@ def setup(ctx): """ Run meson setup """ - # We try to call `--reconfigure` first as a workaround for this issue: + # We add --reconfigure first as a workaround for this issue: # https://github.com/ninja-build/ninja/issues/1997 if MEDIASOUP_BUILDTYPE == 'Release': try: @@ -279,7 +258,7 @@ def mediasoup_worker(ctx): Compile mediasoup-worker binary """ if os.getenv('MEDIASOUP_WORKER_BIN'): - printf('skipping mediasoup-worker compilation due to the existence of the MEDIASOUP_WORKER_BIN environment variable'); + print('skipping mediasoup-worker compilation due to the existence of the MEDIASOUP_WORKER_BIN environment variable'); return; with ctx.cd(WORKER_DIR): @@ -372,14 +351,14 @@ def test(ctx): pty=True ); - MEDIASOUP_TEST_TAGS = os.getenv('MEDIASOUP_TEST_TAGS') or ''; + mediasoup_test_tags = os.getenv('MEDIASOUP_TEST_TAGS') or ''; # On Windows lcov doesn't work (at least not yet) and we need to add .exe to # the binary path. if os.name == 'nt': with ctx.cd(WORKER_DIR): ctx.run( - f'{BUILD_DIR}/mediasoup-worker-test.exe --invisibles --use-colour=yes {MEDIASOUP_TEST_TAGS}', + f'{BUILD_DIR}/mediasoup-worker-test.exe --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, pty=True ); @@ -391,7 +370,7 @@ def test(ctx): ); with ctx.cd(WORKER_DIR): ctx.run( - f'{BUILD_DIR}/mediasoup-worker-test --invisibles --use-colour=yes {MEDIASOUP_TEST_TAGS}', + f'{BUILD_DIR}/mediasoup-worker-test --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, pty=True ); @@ -415,11 +394,11 @@ def test_asan(ctx): pty=True ); - MEDIASOUP_TEST_TAGS = os.getenv('MEDIASOUP_TEST_TAGS') or ''; + mediasoup_test_tags = os.getenv('MEDIASOUP_TEST_TAGS') or ''; with ctx.cd(WORKER_DIR): ctx.run( - f'ASAN_OPTIONS=detect_leaks=1 {BUILD_DIR}/mediasoup-worker-test-asan --invisibles --use-colour=yes {MEDIASOUP_TEST_TAGS}', + f'ASAN_OPTIONS=detect_leaks=1 {BUILD_DIR}/mediasoup-worker-test-asan --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, pty=True ); @@ -430,11 +409,11 @@ def tidy(ctx): """ Perform C++ checks with clang-tidy """ - MEDIASOUP_TIDY_CHECKS = os.getenv('MEDIASOUP_TIDY_CHECKS') or ''; + mediasoup_tidy_checks = os.getenv('MEDIASOUP_TIDY_CHECKS') or ''; with ctx.cd(WORKER_DIR): ctx.run( - f'{PYTHON} ./scripts/clang-tidy.py -clang-tidy-binary=./scripts/node_modules/.bin/clang-tidy -clang-apply-replacements-binary=./scripts/node_modules/.bin/clang-apply-replacements -header-filter="(Channel/**/*.hpp|DepLibSRTP.hpp|DepLibUV.hpp|DepLibWebRTC.hpp|DepOpenSSL.hpp|DepUsrSCTP.hpp|LogLevel.hpp|Logger.hpp|MediaSoupError.hpp|RTC/**/*.hpp|Settings.hpp|Utils.hpp|Worker.hpp|common.hpp|handles/**/*.hpp)" -p={BUILD_DIR} -j={NUM_CORES} -checks={MEDIASOUP_TIDY_CHECKS} -quiet', + f'{PYTHON} ./scripts/clang-tidy.py -clang-tidy-binary=./scripts/node_modules/.bin/clang-tidy -clang-apply-replacements-binary=./scripts/node_modules/.bin/clang-apply-replacements -header-filter="(Channel/**/*.hpp|DepLibSRTP.hpp|DepLibUV.hpp|DepLibWebRTC.hpp|DepOpenSSL.hpp|DepUsrSCTP.hpp|LogLevel.hpp|Logger.hpp|MediaSoupError.hpp|RTC/**/*.hpp|Settings.hpp|Utils.hpp|Worker.hpp|common.hpp|handles/**/*.hpp)" -p={BUILD_DIR} -j={NUM_CORES} -checks={mediasoup_tidy_checks} -quiet', echo=True, pty=True ); @@ -460,7 +439,7 @@ def fuzzer(ctx): @task -def fuzzer(ctx): +def fuzzer_run_all(ctx): """ Run all fuzzer cases """ From 12ba1a31b98edaf18a58f271bf971c8021bda745 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 01:25:53 +0100 Subject: [PATCH 08/35] make 'lint' task also lint tasts.py file itself --- worker/tasks.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 7c4a6d17d6..48e5ae530f 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -1,3 +1,12 @@ +# Ignore these pylint warnings, concentions and refactoring messages: +# - W0301: Unnecessary semicolon (unnecessary-semicolon) +# - W0622: Redefining built-in 'format' (redefined-builtin) +# - W0702: No exception type(s) specified (bare-except) +# - C0114: Missing module docstring (missing-module-docstring) +# - C0301: Line too long (line-too-long) +# +# pylint: disable=W0301,W0622,W0702,C0114,C0301 + # # This is a tasks.py file for the pip invoke package: https://docs.pyinvoke.org/. # @@ -43,6 +52,8 @@ # https://github.com/ninja-build/ninja/issues/2211 # https://github.com/ninja-build/ninja/issues/2212 NINJA_VERSION = os.getenv('NINJA_VERSION') or '1.10.2.4'; +PYLINT = os.getenv('PYLINT') or f'{PIP_DIR}/bin/pylint'; +PYLINT_VERSION = os.getenv('PYLINT_VERSION') or '3.0.2'; NPM = os.getenv('NPM') or 'npm'; LCOV = os.getenv('LCOV') or f'{WORKER_DIR}/deps/lcov/bin/lcov'; DOCKER = os.getenv('DOCKER') or 'docker'; @@ -96,10 +107,10 @@ def meson_ninja(ctx): # https://github.com/NixOS/nixpkgs/issues/142383. pip_build_binaries = '--no-binary :all:' if os.path.isfile('/etc/NIXOS') or os.path.isdir('/etc/guix') else ''; - # Install meson and ninja using pip into our custom location, so we don't - # depend on system-wide installation. + # Install meson, ninja and pylint using pip into our custom location, so we + # don't depend on system-wide installation. ctx.run( - f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} {pip_build_binaries} meson=={MESON_VERSION} ninja=={NINJA_VERSION}', + f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} {pip_build_binaries} meson=={MESON_VERSION} ninja=={NINJA_VERSION} pylint=={PYLINT_VERSION}', echo=True, pty=True ); @@ -160,7 +171,7 @@ def setup(ctx): @task -def clean(ctx): +def clean(ctx): # pylint: disable=unused-argument """ Clean the installation directory """ @@ -171,7 +182,7 @@ def clean(ctx): @task -def clean_build(ctx): +def clean_build(ctx): # pylint: disable=unused-argument """ Clean the build directory """ @@ -182,7 +193,7 @@ def clean_build(ctx): @task -def clean_pip(ctx): +def clean_pip(ctx): # pylint: disable=unused-argument """ Clean the local pip directory """ @@ -307,7 +318,7 @@ def xcode(ctx): ); -@task +@task(pre=[meson_ninja]) def lint(ctx): """ Lint source code @@ -318,6 +329,12 @@ def lint(ctx): echo=True, pty=True ); + with ctx.cd(WORKER_DIR): + ctx.run( + f'{PYLINT} tasks.py', + echo=True, + pty=True + ); @task From d44d81c25c51b62ca153e0d54fe6c37e66fb6790 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 01:32:53 +0100 Subject: [PATCH 09/35] test --- worker/tasks.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/worker/tasks.py b/worker/tasks.py index 48e5ae530f..066a2c740f 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -22,6 +22,9 @@ import shutil; from invoke import task; +# TODO: Temporal to check if we can prevent issue in Windows with pty=True. +print('-------- sys.stdout.isatty():', sys.stdout.isatty()); + MEDIASOUP_BUILDTYPE = os.getenv('MEDIASOUP_BUILDTYPE') or 'Release'; WORKER_DIR = os.path.dirname(os.path.abspath( From 37aeb5671983a912df557bf9a1c00210895ffe85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 01:36:44 +0100 Subject: [PATCH 10/35] goto hell pylint --- worker/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/tasks.py b/worker/tasks.py index 066a2c740f..d6d7230c4e 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -22,7 +22,7 @@ import shutil; from invoke import task; -# TODO: Temporal to check if we can prevent issue in Windows with pty=True. +# TODO: Temporal to check if we can prevent issue in Windows with pty=True. # pylint: disable=fixme print('-------- sys.stdout.isatty():', sys.stdout.isatty()); From a78e61ebf8d360dfac81945dd5b801cfda9f0dfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 01:45:19 +0100 Subject: [PATCH 11/35] let's see if this fixes the "PTY not supported in Windows" --- worker/tasks.py | 80 ++++++++++++++++++++++++------------------------- 1 file changed, 40 insertions(+), 40 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index d6d7230c4e..2ff386f53a 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -22,10 +22,6 @@ import shutil; from invoke import task; -# TODO: Temporal to check if we can prevent issue in Windows with pty=True. # pylint: disable=fixme -print('-------- sys.stdout.isatty():', sys.stdout.isatty()); - - MEDIASOUP_BUILDTYPE = os.getenv('MEDIASOUP_BUILDTYPE') or 'Release'; WORKER_DIR = os.path.dirname(os.path.abspath( inspect.getframeinfo(inspect.currentframe()).filename @@ -60,6 +56,10 @@ NPM = os.getenv('NPM') or 'npm'; LCOV = os.getenv('LCOV') or f'{WORKER_DIR}/deps/lcov/bin/lcov'; DOCKER = os.getenv('DOCKER') or 'docker'; +# pty=True in ctx.run() is not available on Windows so if stdout is not a TTY +# let's assume PTY is not supported. Related issue in invoke project: +# https://github.com/pyinvoke/invoke/issues/561 +PTY_SUPPORTED = sys.stdout.isatty(); # Disable `*.pyc` files creation. os.environ['PYTHONDONTWRITEBYTECODE'] = 'true'; @@ -103,7 +103,7 @@ def meson_ninja(ctx): ctx.run( f'{PYTHON} -m pip install --target={PIP_DIR} pip setuptools', echo=True, - pty=True + pty=PTY_SUPPORTED ); # Workaround for NixOS and Guix that don't work with pre-built binaries, see: @@ -115,7 +115,7 @@ def meson_ninja(ctx): ctx.run( f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} {pip_build_binaries} meson=={MESON_VERSION} ninja=={NINJA_VERSION} pylint=={PYLINT_VERSION}', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -132,14 +132,14 @@ def setup(ctx): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=True + pty=PTY_SUPPORTED ); except: with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=True + pty=PTY_SUPPORTED ); elif MEDIASOUP_BUILDTYPE == 'Debug': try: @@ -147,14 +147,14 @@ def setup(ctx): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=True + pty=PTY_SUPPORTED ); except: with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=True + pty=PTY_SUPPORTED ); else: try: @@ -162,14 +162,14 @@ def setup(ctx): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=True + pty=PTY_SUPPORTED ); except: with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -215,7 +215,7 @@ def clean_subprojects(ctx): ctx.run( f'{MESON} subprojects purge --include-cache --confirm', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -229,7 +229,7 @@ def clean_all(ctx): ctx.run( f'{MESON} subprojects purge --include-cache --confirm', echo=True, - pty=True + pty=PTY_SUPPORTED ); except: pass; @@ -249,7 +249,7 @@ def update_wrap_file(ctx, subproject): ctx.run( f'{MESON} subprojects update --reset {subproject}', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -262,7 +262,7 @@ def flatc(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} flatbuffers-generator', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -279,13 +279,13 @@ def mediasoup_worker(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker', echo=True, - pty=True + pty=PTY_SUPPORTED ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -298,13 +298,13 @@ def libmediasoup_worker(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} libmediasoup-worker', echo=True, - pty=True + pty=PTY_SUPPORTED ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags libmediasoup-worker', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -317,7 +317,7 @@ def xcode(ctx): ctx.run( f'{MESON} setup --buildtype {MEDIASOUP_BUILDTYPE} --backend xcode {MEDIASOUP_OUT_DIR}/xcode', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -330,13 +330,13 @@ def lint(ctx): ctx.run( f'{NPM} run lint --prefix scripts/', echo=True, - pty=True + pty=PTY_SUPPORTED ); with ctx.cd(WORKER_DIR): ctx.run( f'{PYLINT} tasks.py', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -349,7 +349,7 @@ def format(ctx): ctx.run( f'{NPM} run format --prefix scripts/', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -362,13 +362,13 @@ def test(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-test', echo=True, - pty=True + pty=PTY_SUPPORTED ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-test', echo=True, - pty=True + pty=PTY_SUPPORTED ); mediasoup_test_tags = os.getenv('MEDIASOUP_TEST_TAGS') or ''; @@ -380,19 +380,19 @@ def test(ctx): ctx.run( f'{BUILD_DIR}/mediasoup-worker-test.exe --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, - pty=True + pty=PTY_SUPPORTED ); else: ctx.run( f'{LCOV} --directory {WORKER_DIR} --zerocounters', echo=True, - pty=True + pty=PTY_SUPPORTED ); with ctx.cd(WORKER_DIR): ctx.run( f'{BUILD_DIR}/mediasoup-worker-test --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -405,13 +405,13 @@ def test_asan(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-test-asan', echo=True, - pty=True + pty=PTY_SUPPORTED ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-test-asan', echo=True, - pty=True + pty=PTY_SUPPORTED ); mediasoup_test_tags = os.getenv('MEDIASOUP_TEST_TAGS') or ''; @@ -420,7 +420,7 @@ def test_asan(ctx): ctx.run( f'ASAN_OPTIONS=detect_leaks=1 {BUILD_DIR}/mediasoup-worker-test-asan --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -435,7 +435,7 @@ def tidy(ctx): ctx.run( f'{PYTHON} ./scripts/clang-tidy.py -clang-tidy-binary=./scripts/node_modules/.bin/clang-tidy -clang-apply-replacements-binary=./scripts/node_modules/.bin/clang-apply-replacements -header-filter="(Channel/**/*.hpp|DepLibSRTP.hpp|DepLibUV.hpp|DepLibWebRTC.hpp|DepOpenSSL.hpp|DepUsrSCTP.hpp|LogLevel.hpp|Logger.hpp|MediaSoupError.hpp|RTC/**/*.hpp|Settings.hpp|Utils.hpp|Worker.hpp|common.hpp|handles/**/*.hpp)" -p={BUILD_DIR} -j={NUM_CORES} -checks={mediasoup_tidy_checks} -quiet', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -448,13 +448,13 @@ def fuzzer(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-fuzzer', echo=True, - pty=True + pty=PTY_SUPPORTED ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-fuzzer', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -467,7 +467,7 @@ def fuzzer_run_all(ctx): ctx.run( f'LSAN_OPTIONS=verbosity=1:log_threads=1 {BUILD_DIR}/mediasoup-worker-fuzzer -artifact_prefix=fuzzer/reports/ -max_len=1400 fuzzer/new-corpus deps/webrtc-fuzzer-corpora/corpora/stun-corpus deps/webrtc-fuzzer-corpora/corpora/rtp-corpus deps/webrtc-fuzzer-corpora/corpora/rtcp-corpus', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -481,14 +481,14 @@ def docker(ctx): ctx.run( f'{DOCKER} build -f Dockerfile --no-cache --tag mediasoup/docker:latest .', echo=True, - pty=True + pty=PTY_SUPPORTED ); else: with ctx.cd(WORKER_DIR): ctx.run( f'{DOCKER} build -f Dockerfile --tag mediasoup/docker:latest .', echo=True, - pty=True + pty=PTY_SUPPORTED ); @@ -515,14 +515,14 @@ def docker_alpine(ctx): ctx.run( f'{DOCKER} build -f Dockerfile.alpine --no-cache --tag mediasoup/docker-alpine:latest .', echo=True, - pty=True + pty=PTY_SUPPORTED ); else: with ctx.cd(WORKER_DIR): ctx.run( f'{DOCKER} build -f Dockerfile.alpine --tag mediasoup/docker-alpine:latest .', echo=True, - pty=True + pty=PTY_SUPPORTED ); From 4b1a7a16534a3e5a6b49f7365eaf2bb070cb0a07 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 02:03:04 +0100 Subject: [PATCH 12/35] fix some issues --- .github/workflows/{codeql.yml => codeql.yaml} | 3 +++ TODO_INVOKE.md | 17 +++------------ worker/tasks.py | 21 +++++++++++++------ 3 files changed, 21 insertions(+), 20 deletions(-) rename .github/workflows/{codeql.yml => codeql.yaml} (95%) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yaml similarity index 95% rename from .github/workflows/codeql.yml rename to .github/workflows/codeql.yaml index b26646fd42..d6e8799d38 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yaml @@ -55,6 +55,9 @@ jobs: # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality + # TODO: Temporal until we decide how to deal with pip invoke requirement. + - name: pip install invoke + run: python3 -m pip install invoke # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) diff --git a/TODO_INVOKE.md b/TODO_INVOKE.md index 897be87421..28a78f98ce 100644 --- a/TODO_INVOKE.md +++ b/TODO_INVOKE.md @@ -21,20 +21,9 @@ In summary: ### Other TODO items -- `MESON_ARGS` stuff. How to replicate this? What is this `subst` doing exactly? - ```Makefile - ifeq ($(OS),Windows_NT) - ifeq ($(MESON_ARGS),"") - MESON_ARGS = $(subst $\",,"--vsenv") - endif - endif - ``` - -- `build.rs`: Remvoe the `MEDIASOUP_OUT_DIR` env stuff. - - Remove `getmake.py` and related stuff. -- When running meson commands we are loosing colored output, perhaps some invoke `context.run()` option is needed. Reported here: https://github.com/pyinvoke/invoke/discussions/976. Solved using `pty=True` in `ctx.run()` but I need to check the implications. So I'm adding `pty=True` everywehere. Let's see. +- When running meson commands we are loosing colored output, perhaps some invoke `context.run()` option is needed. Reported here: https://github.com/pyinvoke/invoke/discussions/976. Solved using `pty=True` in `ctx.run()` but I need to check the implications. So I'm adding `pty=True` everywehere. Let's see. It seems to work. - Do we still need this in `build.rs`? ```rust @@ -42,8 +31,8 @@ In summary: let mediasoup_out_dir = format!("{}/out", out_dir.replace('\\', "/")); ``` -- "Code QL Analyze (cpp)" CI task fails because "Command failed: invoke -r worker" (as expected): https://github.com/versatica/mediasoup/actions/runs/6973590508/job/18977838894?pr=1239 - - Once we have `invoke` working, let's remove the no longer used `cpu_cores.sh` script. - And of course, remove `Makefile`. + +- Remove the "pip install invoke" step in all CI `.yaml` files. diff --git a/worker/tasks.py b/worker/tasks.py index 2ff386f53a..b2360bde00 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -45,16 +45,16 @@ # worker with tracing and enabled. # NOTE: On Windows make sure to add `--vsenv` or have MSVS environment already # active if you override this parameter. -MESON_ARGS = '--vsenv' if os.name == 'nt' and not os.getenv('MESON_ARGS') else ''; +MESON_ARGS = os.getenv('MESON_ARGS') if os.getenv('MESON_ARGS') else '--vsenv' if os.name == 'nt' else ''; # Let's use a specific version of ninja to avoid buggy version 1.11.1: # https://mediasoup.discourse.group/t/partly-solved-could-not-detect-ninja-v1-8-2-or-newer/ # https://github.com/ninja-build/ninja/issues/2211 # https://github.com/ninja-build/ninja/issues/2212 NINJA_VERSION = os.getenv('NINJA_VERSION') or '1.10.2.4'; -PYLINT = os.getenv('PYLINT') or f'{PIP_DIR}/bin/pylint'; +PYLINT = f'{PIP_DIR}/bin/pylint'; PYLINT_VERSION = os.getenv('PYLINT_VERSION') or '3.0.2'; NPM = os.getenv('NPM') or 'npm'; -LCOV = os.getenv('LCOV') or f'{WORKER_DIR}/deps/lcov/bin/lcov'; +LCOV = f'{WORKER_DIR}/deps/lcov/bin/lcov'; DOCKER = os.getenv('DOCKER') or 'docker'; # pty=True in ctx.run() is not available on Windows so if stdout is not a TTY # let's assume PTY is not supported. Related issue in invoke project: @@ -110,10 +110,10 @@ def meson_ninja(ctx): # https://github.com/NixOS/nixpkgs/issues/142383. pip_build_binaries = '--no-binary :all:' if os.path.isfile('/etc/NIXOS') or os.path.isdir('/etc/guix') else ''; - # Install meson, ninja and pylint using pip into our custom location, so we - # don't depend on system-wide installation. + # Install meson and ninja using pip into our custom location, so we don't + # depend on system-wide installation. ctx.run( - f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} {pip_build_binaries} meson=={MESON_VERSION} ninja=={NINJA_VERSION} pylint=={PYLINT_VERSION}', + f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} {pip_build_binaries} meson=={MESON_VERSION} ninja=={NINJA_VERSION}', echo=True, pty=PTY_SUPPORTED ); @@ -332,6 +332,15 @@ def lint(ctx): echo=True, pty=PTY_SUPPORTED ); + + if not os.path.isfile(PYLINT): + # Install pylint using pip into our custom location. + ctx.run( + f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} pylint=={PYLINT_VERSION}', + echo=True, + pty=PTY_SUPPORTED + ); + with ctx.cd(WORKER_DIR): ctx.run( f'{PYLINT} tasks.py', From 4394329a11432de03e8d91ecd6fae61c9260e610 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 02:26:42 +0100 Subject: [PATCH 13/35] cosmetic --- worker/tasks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/tasks.py b/worker/tasks.py index b2360bde00..5e40015521 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -86,7 +86,7 @@ def meson_ninja(ctx): """ Install meson and ninja (also update Python pip and setuptools packages) """ - if os.path.isdir(PIP_DIR): + if os.path.isfile(MESON): return; # Updated pip and setuptools are needed for meson. From bb3ced0bf8ce4fae1773638818dea5c94c27f7b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 11:29:47 +0100 Subject: [PATCH 14/35] Use different pip folders for groups of packagesd installed separately since `--upgrafe` and `--target` are not compatible because Python (and hence) PIP sucks a lot --- worker/tasks.py | 38 +++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 15 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 5e40015521..f675ab589b 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -30,14 +30,17 @@ MEDIASOUP_OUT_DIR = os.getenv('MEDIASOUP_OUT_DIR') or f'{WORKER_DIR}/out'; MEDIASOUP_INSTALL_DIR = os.getenv('MEDIASOUP_INSTALL_DIR') or f'{MEDIASOUP_OUT_DIR}/{MEDIASOUP_BUILDTYPE}'; BUILD_DIR = os.getenv('BUILD_DIR') or f'{MEDIASOUP_INSTALL_DIR}/build'; -PIP_DIR = f'{MEDIASOUP_OUT_DIR}/pip'; +# Custom pip folder for meson and ninja packages. +PIP_MESON_NINJA_DIR = f'{MEDIASOUP_OUT_DIR}/pip_meson_ninja'; +# Custom pip folder for pylint package. +PIP_PYLINT_DIR = f'{MEDIASOUP_OUT_DIR}/pip_pylint'; # If available (only on some *nix systems), os.sched_getaffinity(0) gets set of # CPUs the calling thread is restricted to. Instead, os.cpu_count() returns the # total number of CPUs in a system (it doesn't take into account how many of them # the calling thread can use). NUM_CORES = len(os.sched_getaffinity(0)) if hasattr(os, 'sched_getaffinity') else os.cpu_count(); PYTHON = os.getenv('PYTHON') or sys.executable; -MESON = os.getenv('MESON') or f'{PIP_DIR}/bin/meson'; +MESON = os.getenv('MESON') or f'{PIP_MESON_NINJA_DIR}/bin/meson'; MESON_VERSION = os.getenv('MESON_VERSION') or '1.2.1'; # MESON_ARGS can be used to provide extra configuration parameters to meson, # such as adding defines or changing optimization options. For instance, use @@ -51,7 +54,7 @@ # https://github.com/ninja-build/ninja/issues/2211 # https://github.com/ninja-build/ninja/issues/2212 NINJA_VERSION = os.getenv('NINJA_VERSION') or '1.10.2.4'; -PYLINT = f'{PIP_DIR}/bin/pylint'; +PYLINT = f'{PIP_PYLINT_DIR}/bin/pylint'; PYLINT_VERSION = os.getenv('PYLINT_VERSION') or '3.0.2'; NPM = os.getenv('NPM') or 'npm'; LCOV = f'{WORKER_DIR}/deps/lcov/bin/lcov'; @@ -67,18 +70,18 @@ # Instruct meson where to look for ninja binary. if os.name == 'nt': # Windows is, of course, special. - os.environ['NINJA'] = f'{PIP_DIR}/bin/ninja.exe'; + os.environ['NINJA'] = f'{PIP_MESON_NINJA_DIR}/bin/ninja.exe'; else: - os.environ['NINJA'] = f'{PIP_DIR}/bin/ninja'; + os.environ['NINJA'] = f'{PIP_MESON_NINJA_DIR}/bin/ninja'; # Instruct Python where to look for modules it needs, such that meson actually # runs from installed location. -# NOTE: For some reason on Windows adding `:{PYTHONPATH}` breaks things. +# NOTE: On Windows we must use ; isntead of : to separate paths. +PYTHONPATH = os.getenv('PYTHONPATH') or ''; if os.name == 'nt': - os.environ['PYTHONPATH'] = PIP_DIR; + os.environ['PYTHONPATH'] = f'{PIP_MESON_NINJA_DIR};{PIP_PYLINT_DIR};{PYTHONPATH}'; else: - PYTHONPATH = os.getenv('PYTHONPATH') or ''; - os.environ['PYTHONPATH'] = f'{PIP_DIR}:{PYTHONPATH}'; + os.environ['PYTHONPATH'] = f'{PIP_MESON_NINJA_DIR}:{PIP_PYLINT_DIR}:{PYTHONPATH}'; @task @@ -95,13 +98,13 @@ def meson_ninja(ctx): # fallback to command without `--system` if the first one fails. try: ctx.run( - f'{PYTHON} -m pip install --system --target={PIP_DIR} pip setuptools', + f'{PYTHON} -m pip install --system --target={PIP_MESON_NINJA_DIR} pip setuptools', echo=True, hide=True ); except: ctx.run( - f'{PYTHON} -m pip install --target={PIP_DIR} pip setuptools', + f'{PYTHON} -m pip install --target={PIP_MESON_NINJA_DIR} pip setuptools', echo=True, pty=PTY_SUPPORTED ); @@ -113,7 +116,7 @@ def meson_ninja(ctx): # Install meson and ninja using pip into our custom location, so we don't # depend on system-wide installation. ctx.run( - f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} {pip_build_binaries} meson=={MESON_VERSION} ninja=={NINJA_VERSION}', + f'{PYTHON} -m pip install --upgrade --target={PIP_MESON_NINJA_DIR} {pip_build_binaries} meson=={MESON_VERSION} ninja=={NINJA_VERSION}', echo=True, pty=PTY_SUPPORTED ); @@ -198,10 +201,15 @@ def clean_build(ctx): # pylint: disable=unused-argument @task def clean_pip(ctx): # pylint: disable=unused-argument """ - Clean the local pip directory + Clean the local pip setup """ try: - shutil.rmtree(PIP_DIR); + shutil.rmtree(PIP_MESON_NINJA_DIR); + except: + pass; + + try: + shutil.rmtree(PIP_PYLINT_DIR); except: pass; @@ -336,7 +344,7 @@ def lint(ctx): if not os.path.isfile(PYLINT): # Install pylint using pip into our custom location. ctx.run( - f'{PYTHON} -m pip install --upgrade --target={PIP_DIR} pylint=={PYLINT_VERSION}', + f'{PYTHON} -m pip install --upgrade --target={PIP_PYLINT_DIR} pylint=={PYLINT_VERSION}', echo=True, pty=PTY_SUPPORTED ); From f4c46a1a4d58eac2504772256c07e23d922116a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 12:05:52 +0100 Subject: [PATCH 15/35] Remove TODO_INVOKE.md --- TODO_INVOKE.md | 38 -------------------------------------- 1 file changed, 38 deletions(-) delete mode 100644 TODO_INVOKE.md diff --git a/TODO_INVOKE.md b/TODO_INVOKE.md deleted file mode 100644 index 28a78f98ce..0000000000 --- a/TODO_INVOKE.md +++ /dev/null @@ -1,38 +0,0 @@ -# TODO pip invoke - -### pip `invoke` package is required - -This solution requires having the pip invoke package installed, however I think -we could just include it in the mediasoup repo in the worker/pip-invoke folder -and make npm-scripts.mjs invoke its executable in worker/pip-invoke/bin/invoke -(same for Rust). However that's hard since we also need to later override THONPATH -env for pip invoke module to be found. So... -Perhaps it's easier if we run worker/scripts/get-pip-invoke.sh during the -installation. That script installs pip invoke package in default pip location. -Or we can set PYTHONPATH env within npm-scripts.mjs (and Rust) and also in the -Makefile, so we don't remove it and instead make it behave as a proxy to invoke. -Let's see. - -In summary: -- `build.rs` needs to call `invoke` instead of `make`. Should we mandate `pip install invoke` when using mediasoup-rust? -- Same for `npm-scripts.mjs` which now should call `invoke` instead of `make`. -- Check if current `worker/scripts/get-pip-invoke.sh` script makes sense. - - -### Other TODO items - -- Remove `getmake.py` and related stuff. - -- When running meson commands we are loosing colored output, perhaps some invoke `context.run()` option is needed. Reported here: https://github.com/pyinvoke/invoke/discussions/976. Solved using `pty=True` in `ctx.run()` but I need to check the implications. So I'm adding `pty=True` everywehere. Let's see. It seems to work. - -- Do we still need this in `build.rs`? - ```rust - // Force forward slashes on Windows too so that is plays well with our dumb `Makefile`. - let mediasoup_out_dir = format!("{}/out", out_dir.replace('\\', "/")); - ``` - -- Once we have `invoke` working, let's remove the no longer used `cpu_cores.sh` script. - -- And of course, remove `Makefile`. - -- Remove the "pip install invoke" step in all CI `.yaml` files. From af9f836079391181213f1d267c37f3685e9e44c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 12:06:06 +0100 Subject: [PATCH 16/35] fix typo --- worker/tasks.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/worker/tasks.py b/worker/tasks.py index f675ab589b..e683573a59 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -33,6 +33,8 @@ # Custom pip folder for meson and ninja packages. PIP_MESON_NINJA_DIR = f'{MEDIASOUP_OUT_DIR}/pip_meson_ninja'; # Custom pip folder for pylint package. +# NOTE: We do this because using --target and --upgrade in `pip install` is not +# supported and a latter invokation entirely replaces the bin folder. PIP_PYLINT_DIR = f'{MEDIASOUP_OUT_DIR}/pip_pylint'; # If available (only on some *nix systems), os.sched_getaffinity(0) gets set of # CPUs the calling thread is restricted to. Instead, os.cpu_count() returns the @@ -76,7 +78,7 @@ # Instruct Python where to look for modules it needs, such that meson actually # runs from installed location. -# NOTE: On Windows we must use ; isntead of : to separate paths. +# NOTE: On Windows we must use ; instead of : to separate paths. PYTHONPATH = os.getenv('PYTHONPATH') or ''; if os.name == 'nt': os.environ['PYTHONPATH'] = f'{PIP_MESON_NINJA_DIR};{PIP_PYLINT_DIR};{PYTHONPATH}'; From 8ccfb2713e77a18303e99447718c61338a40f692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 13:18:44 +0100 Subject: [PATCH 17/35] TMP: Install invoke in Dockerfiles --- worker/Dockerfile | 16 +++++++++------- worker/Dockerfile.alpine | 12 +++++++----- 2 files changed, 16 insertions(+), 12 deletions(-) diff --git a/worker/Dockerfile b/worker/Dockerfile index df9193063d..6435f2442d 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -1,15 +1,12 @@ FROM ubuntu:22.04 # Install dependencies. -RUN \ - set -x \ +RUN set -x \ && apt-get update \ && apt-get install --yes \ - bash-completion wget curl subversion screen gcc g++ cmake ninja-build golang \ - autoconf libtool apache2 python3-pip python3-dev pkg-config zlib1g-dev \ - libgss-dev libssl-dev libxml2-dev nasm libarchive-dev make automake \ - libdbus-1-dev libboost-dev autoconf-archive bash-completion python3-yaml \ - clang + gcc g++ clang pkg-config bash-completion wget curl \ + screen python3-pip python3-yaml pkg-config zlib1g-dev \ + libgss-dev libssl-dev libxml2-dev # Install node 20. RUN set -x \ @@ -29,6 +26,11 @@ ENV LANG="C.UTF-8" ENV CC="clang" ENV CXX="clang++" +# Install pip invoke package. +# TODO: Temporal. +RUN set -x \ + && pip3 install invoke + WORKDIR /mediasoup CMD ["bash"] diff --git a/worker/Dockerfile.alpine b/worker/Dockerfile.alpine index 5ae43d88c1..f12546304b 100644 --- a/worker/Dockerfile.alpine +++ b/worker/Dockerfile.alpine @@ -1,17 +1,19 @@ FROM alpine # Install dependencies. -RUN \ - set -x \ - && apk add \ - nodejs-current npm python3 py3-pip \ - make gcc g++ +RUN set -x \ + && apk add gcc g++ nodejs-current npm python3 py3-pip # Make CC and CXX point to gcc/g++. ENV LANG="C.UTF-8" ENV CC="gcc" ENV CXX="g++" +# Install pip invoke package. +# TODO: Temporal. +RUN set -x \ + && pip3 install invoke + WORKDIR /mediasoup CMD ["ash"] From 36a5e90414a9932e7b509c1fb161906716700228 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 13:37:01 +0100 Subject: [PATCH 18/35] Install pip invoke in a custom location during npm installation and keep Makefile as a proxy to invoke (WIP) --- .github/workflows/mediasoup-node.yaml | 4 - .gitignore | 5 +- npm-scripts.mjs | 47 ++++- worker/Dockerfile | 5 - worker/Dockerfile.alpine | 5 - worker/Makefile | 275 ++++++-------------------- worker/tasks.py | 3 +- 7 files changed, 112 insertions(+), 232 deletions(-) diff --git a/.github/workflows/mediasoup-node.yaml b/.github/workflows/mediasoup-node.yaml index ae5ab5da51..f22c707306 100644 --- a/.github/workflows/mediasoup-node.yaml +++ b/.github/workflows/mediasoup-node.yaml @@ -47,10 +47,6 @@ jobs: restore-keys: | ${{ matrix.ci.os }}-node- - # TODO: Temporal until we decide how to deal with pip invoke requirement. - - name: pip install invoke - run: python3 -m pip install invoke - # NOTE: Add --force since otherwise `npm ci` fails in Node 16 because # @octokit/auth-token dev dependency requires Node >= 16. - name: npm ci diff --git a/.gitignore b/.gitignore index abbaa0f7f2..e2f65fca29 100644 --- a/.gitignore +++ b/.gitignore @@ -20,10 +20,11 @@ # Flatc generated files. /worker/include/FBS /worker/prebuild -# Vistual Studio generated Stuff. +# Python invoke. +/worker/pip_invoke +# Build artifacts. /worker/**/Debug /worker/**/Release -/worker/.vs # clang-fuzzer stuff is too big. /worker/deps/clang-fuzzer # Ignore all fuzzer generated test inputs. diff --git a/npm-scripts.mjs b/npm-scripts.mjs index c4feea35d1..343d01b770 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -9,7 +9,9 @@ import tar from 'tar'; const PKG = JSON.parse(fs.readFileSync('./package.json').toString()); const IS_WINDOWS = os.platform() === 'win32'; const MAYOR_VERSION = PKG.version.split('.')[0]; -const INVOKE = process.env.INVOKE || 'invoke'; +const PIP_INVOKE_DIR = 'worker/pip_invoke'; +const INVOKE = `${PIP_INVOKE_DIR}/bin/invoke`; +const INVOKE_VERSION = process.env.INVOKE_VERSION ?? '2.0.0'; const FLATBUFFERS_VERSION = '23.3.3'; const WORKER_RELEASE_DIR = 'worker/out/Release'; const WORKER_RELEASE_BIN = IS_WINDOWS ? 'mediasoup-worker.exe' : 'mediasoup-worker'; @@ -22,6 +24,17 @@ const GH_REPO = 'mediasoup'; const task = process.argv.slice(2).join(' '); +// PYTHONPATH env must be updated now so all invoke calls below will find the +// pip invoke module. +if (IS_WINDOWS) +{ + process.env.PYTHONPATH = `${PIP_INVOKE_DIR};${process.env.PYTHONPATH}`; +} +else +{ + process.env.PYTHONPATH = `${PIP_INVOKE_DIR}:${process.env.PYTHONPATH}`; +} + run(); async function run() @@ -273,6 +286,37 @@ async function run() } } +function installInvoke() +{ + if (fs.existsSync(PIP_INVOKE_DIR)) + { + return; + } + + logInfo('installInvoke()'); + + let python = process.env.PYTHON; + + if (!python) + { + try + { + execSync('python3 -version', { stdio: [ 'ignore', 'ignore', 'ignore' ] }); + python = 'python3'; + } + catch (error) + { + python = 'python'; + } + } + + // Install `invoke` into custom location, so we don't depend on system-wide + // installation. + executeCmd( + `${python} -m pip install --upgrade --target=${PIP_INVOKE_DIR} invoke==${INVOKE_VERSION}`, /* exitOnError */ true + ); +} + function deleteNodeLib() { if (!fs.existsSync('node/lib')) @@ -310,6 +354,7 @@ function buildWorker() { logInfo('buildWorker()'); + installInvoke(); executeCmd(`${INVOKE} -r worker`); } diff --git a/worker/Dockerfile b/worker/Dockerfile index 6435f2442d..81f4294115 100644 --- a/worker/Dockerfile +++ b/worker/Dockerfile @@ -26,11 +26,6 @@ ENV LANG="C.UTF-8" ENV CC="clang" ENV CXX="clang++" -# Install pip invoke package. -# TODO: Temporal. -RUN set -x \ - && pip3 install invoke - WORKDIR /mediasoup CMD ["bash"] diff --git a/worker/Dockerfile.alpine b/worker/Dockerfile.alpine index f12546304b..d1b31a6062 100644 --- a/worker/Dockerfile.alpine +++ b/worker/Dockerfile.alpine @@ -9,11 +9,6 @@ ENV LANG="C.UTF-8" ENV CC="gcc" ENV CXX="g++" -# Install pip invoke package. -# TODO: Temporal. -RUN set -x \ - && pip3 install invoke - WORKDIR /mediasoup CMD ["ash"] diff --git a/worker/Makefile b/worker/Makefile index b7e0faed02..54a238ee41 100644 --- a/worker/Makefile +++ b/worker/Makefile @@ -1,72 +1,25 @@ # # make tasks for mediasoup-worker. # +# NOTE: This Makefile is a proxy to pip invoke commands. +# # We need Python 3 here. PYTHON ?= $(shell command -v python3 2> /dev/null || echo python) -WORKER_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) -NUM_CORES ?= $(shell ${WORKER_DIR}/scripts/cpu_cores.sh || echo 4) -MEDIASOUP_OUT_DIR ?= $(shell pwd)/out -# Controls build types, `Release` and `Debug` are presets optimized for those -# use cases. Other build types are possible too, but they are not presets and -# will require `MESON_ARGS` use to customize build configuration. -# Check the meaning of useful macros in the `worker/include/Logger.hpp` header -# file if you want to enable tracing or other debug information. -MEDIASOUP_BUILDTYPE ?= Release -LCOV = $(WORKER_DIR)/deps/lcov/bin/lcov -DOCKER ?= docker -PIP_DIR = $(MEDIASOUP_OUT_DIR)/pip -MEDIASOUP_INSTALL_DIR ?= $(MEDIASOUP_OUT_DIR)/$(MEDIASOUP_BUILDTYPE) -BUILD_DIR ?= $(MEDIASOUP_INSTALL_DIR)/build -MESON ?= $(PIP_DIR)/bin/meson -MESON_VERSION ?= 1.2.1 -# `MESON_ARGS` can be used to provide extra configuration parameters to Meson, -# such as adding defines or changing optimization options. For instance, use -# `MESON_ARGS="-Dms_log_trace=true -Dms_log_file_line=true" npm i` to compile -# worker with tracing and enabled. -# -# NOTE: On Windows make sure to add `--vsenv` or have MSVS environment already -# active if you override this parameter. -MESON_ARGS ?= "" -# Workaround for NixOS and Guix that don't work with pre-built binaries, see: -# https://github.com/NixOS/nixpkgs/issues/142383. -PIP_BUILD_BINARIES = $(shell [ -f /etc/NIXOS -o -d /etc/guix ] && echo "--no-binary :all:") -# Let's use a specific version of ninja to avoid buggy version 1.11.1: -# https://mediasoup.discourse.group/t/partly-solved-could-not-detect-ninja-v1-8-2-or-newer/ -# https://github.com/ninja-build/ninja/issues/2211 -# https://github.com/ninja-build/ninja/issues/2212 -NINJA_VERSION ?= 1.10.2.4 -NPM ?= npm +PIP_INVOKE_DIR = $(shell pwd)/pip_invoke +INVOKE = $(PIP_INVOKE_DIR)/bin/invoke +INVOKE_VERSION ?= 2.0.0 -# Disable `*.pyc` files creation. -export PYTHONDONTWRITEBYTECODE = 1 -# Instruct `meson` where to look for ninja binary. -ifeq ($(OS),Windows_NT) - # Windows is, of course, special. - export NINJA = $(PIP_DIR)/bin/ninja.exe -else - export NINJA = $(PIP_DIR)/bin/ninja -endif - -# Instruct Python where to look for modules it needs, such that `meson` actually -# runs from installed location. -# -# NOTE: For some reason on Windows adding `:${PYTHONPATH}` breaks things. +# Instruct Python where to look for invoke module. ifeq ($(OS),Windows_NT) - export PYTHONPATH := $(PIP_DIR) + export PYTHONPATH := $(PIP_INVOKE_DIR);${PYTHONPATH} else - export PYTHONPATH := $(PIP_DIR):${PYTHONPATH} -endif - -# Activate VS environment on Windows by default. -ifeq ($(OS),Windows_NT) -ifeq ($(MESON_ARGS),"") - MESON_ARGS = $(subst $\",,"--vsenv") -endif + export PYTHONPATH := $(PIP_INVOKE_DIR):${PYTHONPATH} endif .PHONY: \ default \ + invoke \ meson-ninja \ setup \ clean \ @@ -93,7 +46,13 @@ endif default: mediasoup-worker -meson-ninja: +invoke: +ifeq ($(wildcard $(PIP_INVOKE_DIR)),) + # Install `invoke` into custom location, so we don't depend on system-wide installation. + $(PYTHON) -m pip install --upgrade --target=$(PIP_INVOKE_DIR) invoke==$(INVOKE_VERSION) +endif + +meson-ninja: invoke ifeq ($(wildcard $(PIP_DIR)),) # Updated pip and setuptools are needed for meson. # `--system` is not present everywhere and is only needed as workaround for @@ -108,181 +67,69 @@ ifeq ($(wildcard $(PIP_DIR)),) $(PYTHON) -m pip install --upgrade --target=$(PIP_DIR) $(PIP_BUILD_BINARIES) meson==$(MESON_VERSION) ninja==$(NINJA_VERSION) endif -setup: meson-ninja -# We try to call `--reconfigure` first as a workaround for this issue: -# https://github.com/ninja-build/ninja/issues/1997 -ifeq ($(MEDIASOUP_BUILDTYPE),Release) - $(MESON) setup \ - --prefix $(MEDIASOUP_INSTALL_DIR) \ - --bindir "" \ - --libdir "" \ - --buildtype release \ - -Db_ndebug=true \ - -Db_pie=true \ - -Db_staticpic=true \ - --reconfigure \ - $(MESON_ARGS) \ - $(BUILD_DIR) || \ - $(MESON) setup \ - --prefix $(MEDIASOUP_INSTALL_DIR) \ - --bindir "" \ - --libdir "" \ - --buildtype release \ - -Db_ndebug=true \ - -Db_pie=true \ - -Db_staticpic=true \ - $(MESON_ARGS) \ - $(BUILD_DIR) -else -ifeq ($(MEDIASOUP_BUILDTYPE),Debug) - $(MESON) setup \ - --prefix $(MEDIASOUP_INSTALL_DIR) \ - --bindir "" \ - --libdir "" \ - --buildtype debug \ - -Db_pie=true \ - -Db_staticpic=true \ - --reconfigure \ - $(MESON_ARGS) \ - $(BUILD_DIR) || \ - $(MESON) setup \ - --prefix $(MEDIASOUP_INSTALL_DIR) \ - --bindir "" \ - --libdir "" \ - --buildtype debug \ - -Db_pie=true \ - -Db_staticpic=true \ - $(MESON_ARGS) \ - $(BUILD_DIR) -else - $(MESON) setup \ - --prefix $(MEDIASOUP_INSTALL_DIR) \ - --bindir "" \ - --libdir "" \ - --buildtype $(MEDIASOUP_BUILDTYPE) \ - -Db_ndebug=if-release \ - -Db_pie=true \ - -Db_staticpic=true \ - --reconfigure \ - $(MESON_ARGS) \ - $(BUILD_DIR) || \ - $(MESON) setup \ - --prefix $(MEDIASOUP_INSTALL_DIR) \ - --bindir "" \ - --libdir "" \ - --buildtype $(MEDIASOUP_BUILDTYPE) \ - -Db_ndebug=if-release \ - -Db_pie=true \ - -Db_staticpic=true \ - $(MESON_ARGS) \ - $(BUILD_DIR) -endif -endif +setup: invoke + $(INVOKE) setup -clean: - $(RM) -rf $(MEDIASOUP_INSTALL_DIR) +clean: invoke + $(INVOKE) clean -clean-build: - $(RM) -rf $(BUILD_DIR) +clean-build: invoke + $(INVOKE) clean-build -clean-pip: - $(RM) -rf $(PIP_DIR) +clean-pip: invoke + $(INVOKE) clean-pip -clean-subprojects: meson-ninja - $(MESON) subprojects purge --include-cache --confirm +clean-subprojects: invoke + $(INVOKE) clean-subprojects -# Clean subprojects (ignore if it fails) and delete out dir. -clean-all: - -$(MESON) subprojects purge --include-cache --confirm - $(RM) -rf $(MEDIASOUP_OUT_DIR) +clean-all: invoke + $(INVOKE) clean-all -# Update the wrap file of a subproject. Usage example: -# make update-wrap-file SUBPROJECT=openssl -update-wrap-file: meson-ninja - $(MESON) subprojects update --reset $(SUBPROJECT) +# It requires the SUBPROJECT environment variable. +update-wrap-file: invoke + $(INVOKE) subprojects $(SUBPROJECT) -mediasoup-worker: setup flatc -ifeq ($(MEDIASOUP_WORKER_BIN),) - $(MESON) compile -C $(BUILD_DIR) -j $(NUM_CORES) mediasoup-worker - $(MESON) install -C $(BUILD_DIR) --no-rebuild --tags mediasoup-worker -endif +mediasoup-worker: invoke + $(INVOKE) mediasoup-worker -libmediasoup-worker: setup flatc - $(MESON) compile -C $(BUILD_DIR) -j $(NUM_CORES) libmediasoup-worker - $(MESON) install -C $(BUILD_DIR) --no-rebuild --tags libmediasoup-worker +libmediasoup-worker: invoke + $(INVOKE) libmediasoup-worker -flatc: setup - $(MESON) compile -C $(BUILD_DIR) flatbuffers-generator +flatc: invoke + $(INVOKE) flatc -xcode: setup flatc - $(MESON) setup --buildtype debug --backend xcode $(MEDIASOUP_OUT_DIR)/xcode +xcode: invoke + $(INVOKE) xcode -lint: - $(NPM) run lint --prefix scripts/ +lint: invoke + $(INVOKE) lint -format: - $(NPM) run format --prefix scripts/ +format: invoke + $(INVOKE) format -test: setup flatc - $(MESON) compile -C $(BUILD_DIR) -j $(NUM_CORES) mediasoup-worker-test - $(MESON) install -C $(BUILD_DIR) --no-rebuild --tags mediasoup-worker-test -ifeq ($(OS),Windows_NT) - # On Windows lcov doesn't work (at least not yet) and we need to add `.exe` to - # the binary path. - $(BUILD_DIR)/mediasoup-worker-test.exe --invisibles --use-colour=yes $(MEDIASOUP_TEST_TAGS) -else - $(LCOV) --directory ./ --zerocounters - $(BUILD_DIR)/mediasoup-worker-test --invisibles --use-colour=yes $(MEDIASOUP_TEST_TAGS) -endif +test: invoke + $(INVOKE) test -test-asan: setup flatc - $(MESON) compile -C $(BUILD_DIR) -j $(NUM_CORES) mediasoup-worker-test-asan - $(MESON) install -C $(BUILD_DIR) --no-rebuild --tags mediasoup-worker-test-asan - ASAN_OPTIONS=detect_leaks=1 $(BUILD_DIR)/mediasoup-worker-test-asan --invisibles --use-colour=yes $(MEDIASOUP_TEST_TAGS) +test-asan: invoke + $(INVOKE) test-asan -tidy: - $(PYTHON) ./scripts/clang-tidy.py \ - -clang-tidy-binary=./scripts/node_modules/.bin/clang-tidy \ - -clang-apply-replacements-binary=./scripts/node_modules/.bin/clang-apply-replacements \ - -header-filter="(Channel/**/*.hpp|DepLibSRTP.hpp|DepLibUV.hpp|DepLibWebRTC.hpp|DepOpenSSL.hpp|DepUsrSCTP.hpp|LogLevel.hpp|Logger.hpp|MediaSoupError.hpp|RTC/**/*.hpp|Settings.hpp|Utils.hpp|Worker.hpp|common.hpp|handles/**/*.hpp)" \ - -p=$(BUILD_DIR) \ - -j=$(NUM_CORES) \ - -checks=$(MEDIASOUP_TIDY_CHECKS) \ - -quiet +tidy: invoke + $(INVOKE) tidy -fuzzer: setup flatc - $(MESON) compile -C $(BUILD_DIR) -j $(NUM_CORES) mediasoup-worker-fuzzer - $(MESON) install -C $(BUILD_DIR) --no-rebuild --tags mediasoup-worker-fuzzer +fuzzer: invoke + $(INVOKE) fuzzer -fuzzer-run-all: - LSAN_OPTIONS=verbosity=1:log_threads=1 $(BUILD_DIR)/mediasoup-worker-fuzzer -artifact_prefix=fuzzer/reports/ -max_len=1400 fuzzer/new-corpus deps/webrtc-fuzzer-corpora/corpora/stun-corpus deps/webrtc-fuzzer-corpora/corpora/rtp-corpus deps/webrtc-fuzzer-corpora/corpora/rtcp-corpus +fuzzer-run-all: invoke + $(INVOKE) fuzzer-run-all -docker: -ifeq ($(DOCKER_NO_CACHE),true) - $(DOCKER) build -f Dockerfile --no-cache --tag mediasoup/docker:latest . -else - $(DOCKER) build -f Dockerfile --tag mediasoup/docker:latest . -endif +docker: invoke + $(INVOKE) docker -docker-run: - $(DOCKER) run \ - --name=mediasoupDocker -it --rm \ - --privileged \ - --cap-add SYS_PTRACE \ - -v $(WORKER_DIR)/../:/mediasoup \ - mediasoup/docker:latest +docker-run: invoke + $(INVOKE) docker-run -docker-alpine: -ifeq ($(DOCKER_NO_CACHE),true) - $(DOCKER) build -f Dockerfile.alpine --no-cache --tag mediasoup/docker-alpine:latest . -else - $(DOCKER) build -f Dockerfile.alpine --tag mediasoup/docker-alpine:latest . -endif +docker-alpine: invoke + $(INVOKE) docker-alpine -docker-alpine-run: - $(DOCKER) run \ - --name=mediasoupDockerAlpine -it --rm \ - --privileged \ - --cap-add SYS_PTRACE \ - -v $(WORKER_DIR)/../:/mediasoup \ - mediasoup/docker-alpine:latest +docker-alpine-run: invoke + $(INVOKE) docker-alpine-run diff --git a/worker/tasks.py b/worker/tasks.py index e683573a59..9cbb171a06 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -20,7 +20,8 @@ import os; import inspect; import shutil; -from invoke import task; +# We import this from a custom location and pylint doesn't know. +from invoke import task; # pylint: disable=import-error MEDIASOUP_BUILDTYPE = os.getenv('MEDIASOUP_BUILDTYPE') or 'Release'; WORKER_DIR = os.path.dirname(os.path.abspath( From f1041402ac6419503fd99120984a7a2ce94da399 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 13:43:45 +0100 Subject: [PATCH 19/35] fix super error --- npm-scripts.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npm-scripts.mjs b/npm-scripts.mjs index 343d01b770..cf3e9cfc37 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -301,7 +301,7 @@ function installInvoke() { try { - execSync('python3 -version', { stdio: [ 'ignore', 'ignore', 'ignore' ] }); + execSync('python3 --version', { stdio: [ 'ignore', 'ignore', 'ignore' ] }); python = 'python3'; } catch (error) From d93c4377cb65199702eb0132033195ce1be51444 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 13:57:35 +0100 Subject: [PATCH 20/35] Add npm "preinstall" script and install invoke there --- .github/workflows/mediasoup-worker.yaml | 4 ---- npm-scripts.mjs | 8 +++++++- package.json | 1 + 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/mediasoup-worker.yaml b/.github/workflows/mediasoup-worker.yaml index 717bf2df68..f4e3bc16b6 100644 --- a/.github/workflows/mediasoup-worker.yaml +++ b/.github/workflows/mediasoup-worker.yaml @@ -64,10 +64,6 @@ jobs: restore-keys: | ${{ matrix.build.os }}-node-${{matrix.build.cc}}- - # TODO: Temporal until we decide how to deal with pip invoke requirement. - - name: pip install invoke - run: python3 -m pip install invoke - # We need to install some NPM production deps for npm-scripts.mjs to work. - name: npm ci run: npm ci --ignore-scripts --omit=dev diff --git a/npm-scripts.mjs b/npm-scripts.mjs index cf3e9cfc37..28639cd75c 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -41,6 +41,13 @@ async function run() { switch (task) { + case 'preinstall': + { + installInvoke(); + + break; + } + // As per NPM documentation (https://docs.npmjs.com/cli/v9/using-npm/scripts) // `prepare` script: // @@ -354,7 +361,6 @@ function buildWorker() { logInfo('buildWorker()'); - installInvoke(); executeCmd(`${INVOKE} -r worker`); } diff --git a/package.json b/package.json index 9aa28d4a5b..83c681a6e6 100644 --- a/package.json +++ b/package.json @@ -50,6 +50,7 @@ "nodejs" ], "scripts": { + "preinstall": "node npm-scripts.mjs preinstall", "prepare": "node npm-scripts.mjs prepare", "postinstall": "node npm-scripts.mjs postinstall", "typescript:build": "node npm-scripts.mjs typescript:build", From 1af8e0974dfa2853621efc09792c3c0f826de97c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 14:04:50 +0100 Subject: [PATCH 21/35] This will be better in CI --- .github/workflows/codeql.yaml | 6 +++--- .github/workflows/mediasoup-worker-prebuild.yaml | 9 +++++---- .github/workflows/mediasoup-worker.yaml | 5 +++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index d6e8799d38..0055d4ba4e 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -55,9 +55,9 @@ jobs: # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality - # TODO: Temporal until we decide how to deal with pip invoke requirement. - - name: pip install invoke - run: python3 -m pip install invoke + # We need to install pip invoke by manually invoking the NPM 'preinstall' script. + - name: npm run preinstall + run: npm run preinstall # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) diff --git a/.github/workflows/mediasoup-worker-prebuild.yaml b/.github/workflows/mediasoup-worker-prebuild.yaml index d9d34d91eb..7260e2fca4 100644 --- a/.github/workflows/mediasoup-worker-prebuild.yaml +++ b/.github/workflows/mediasoup-worker-prebuild.yaml @@ -50,14 +50,15 @@ jobs: with: node-version: ${{ matrix.node }} - # TODO: Temporal until we decide how to deal with pip invoke requirement. - - name: pip install invoke - run: python3 -m pip install invoke - # We need to install some NPM production deps for npm-scripts.mjs to work. - name: npm ci run: npm ci --ignore-scripts --omit=dev + # However we also need to install pip invoke by manually invoking the NPM + # 'preinstall' script (since `--ignore-scripts` above made it not run). + - name: npm run preinstall + run: npm run preinstall + - name: npm run worker:build run: npm run worker:build diff --git a/.github/workflows/mediasoup-worker.yaml b/.github/workflows/mediasoup-worker.yaml index f4e3bc16b6..304e0741c0 100644 --- a/.github/workflows/mediasoup-worker.yaml +++ b/.github/workflows/mediasoup-worker.yaml @@ -68,6 +68,11 @@ jobs: - name: npm ci run: npm ci --ignore-scripts --omit=dev + # However we also need to install pip invoke by manually invoking the NPM + # 'preinstall' script (since `--ignore-scripts` above made it not run). + - name: npm run preinstall + run: npm run preinstall + # We need to install deps on worker/scripts/package.json. - name: npm run install-worker-dev-tools run: npm run install-worker-dev-tools From 485e240ec8a3ede47b93166e2af293113d9fcb86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Fri, 24 Nov 2023 14:39:56 +0100 Subject: [PATCH 22/35] tasks.py: use sh instead of bash in *unix/macOS since it's more available (example in Alpine Linux) --- worker/tasks.py | 122 ++++++++++++++++++++++++++++++++---------------- 1 file changed, 82 insertions(+), 40 deletions(-) diff --git a/worker/tasks.py b/worker/tasks.py index 9cbb171a06..4b4e3d4687 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -66,6 +66,8 @@ # let's assume PTY is not supported. Related issue in invoke project: # https://github.com/pyinvoke/invoke/issues/561 PTY_SUPPORTED = sys.stdout.isatty(); +# Use sh (widely supported, more than bash) if not in Windows. +SHELL = '/bin/sh' if not os.name == 'nt' else None; # Disable `*.pyc` files creation. os.environ['PYTHONDONTWRITEBYTECODE'] = 'true'; @@ -103,13 +105,15 @@ def meson_ninja(ctx): ctx.run( f'{PYTHON} -m pip install --system --target={PIP_MESON_NINJA_DIR} pip setuptools', echo=True, - hide=True + hide=True, + shell=SHELL ); except: ctx.run( f'{PYTHON} -m pip install --target={PIP_MESON_NINJA_DIR} pip setuptools', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); # Workaround for NixOS and Guix that don't work with pre-built binaries, see: @@ -121,7 +125,8 @@ def meson_ninja(ctx): ctx.run( f'{PYTHON} -m pip install --upgrade --target={PIP_MESON_NINJA_DIR} {pip_build_binaries} meson=={MESON_VERSION} ninja=={NINJA_VERSION}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -138,14 +143,16 @@ def setup(ctx): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); except: with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); elif MEDIASOUP_BUILDTYPE == 'Debug': try: @@ -153,14 +160,16 @@ def setup(ctx): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); except: with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); else: try: @@ -168,14 +177,16 @@ def setup(ctx): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); except: with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -226,7 +237,8 @@ def clean_subprojects(ctx): ctx.run( f'{MESON} subprojects purge --include-cache --confirm', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -240,7 +252,8 @@ def clean_all(ctx): ctx.run( f'{MESON} subprojects purge --include-cache --confirm', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); except: pass; @@ -260,7 +273,8 @@ def update_wrap_file(ctx, subproject): ctx.run( f'{MESON} subprojects update --reset {subproject}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -273,7 +287,8 @@ def flatc(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} flatbuffers-generator', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -290,13 +305,15 @@ def mediasoup_worker(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -309,13 +326,15 @@ def libmediasoup_worker(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} libmediasoup-worker', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags libmediasoup-worker', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -328,7 +347,8 @@ def xcode(ctx): ctx.run( f'{MESON} setup --buildtype {MEDIASOUP_BUILDTYPE} --backend xcode {MEDIASOUP_OUT_DIR}/xcode', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -341,7 +361,8 @@ def lint(ctx): ctx.run( f'{NPM} run lint --prefix scripts/', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); if not os.path.isfile(PYLINT): @@ -349,14 +370,16 @@ def lint(ctx): ctx.run( f'{PYTHON} -m pip install --upgrade --target={PIP_PYLINT_DIR} pylint=={PYLINT_VERSION}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( f'{PYLINT} tasks.py', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -369,7 +392,8 @@ def format(ctx): ctx.run( f'{NPM} run format --prefix scripts/', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -382,13 +406,15 @@ def test(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-test', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-test', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); mediasoup_test_tags = os.getenv('MEDIASOUP_TEST_TAGS') or ''; @@ -400,19 +426,22 @@ def test(ctx): ctx.run( f'{BUILD_DIR}/mediasoup-worker-test.exe --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); else: ctx.run( f'{LCOV} --directory {WORKER_DIR} --zerocounters', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( f'{BUILD_DIR}/mediasoup-worker-test --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -425,13 +454,15 @@ def test_asan(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-test-asan', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-test-asan', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); mediasoup_test_tags = os.getenv('MEDIASOUP_TEST_TAGS') or ''; @@ -440,7 +471,8 @@ def test_asan(ctx): ctx.run( f'ASAN_OPTIONS=detect_leaks=1 {BUILD_DIR}/mediasoup-worker-test-asan --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -455,7 +487,8 @@ def tidy(ctx): ctx.run( f'{PYTHON} ./scripts/clang-tidy.py -clang-tidy-binary=./scripts/node_modules/.bin/clang-tidy -clang-apply-replacements-binary=./scripts/node_modules/.bin/clang-apply-replacements -header-filter="(Channel/**/*.hpp|DepLibSRTP.hpp|DepLibUV.hpp|DepLibWebRTC.hpp|DepOpenSSL.hpp|DepUsrSCTP.hpp|LogLevel.hpp|Logger.hpp|MediaSoupError.hpp|RTC/**/*.hpp|Settings.hpp|Utils.hpp|Worker.hpp|common.hpp|handles/**/*.hpp)" -p={BUILD_DIR} -j={NUM_CORES} -checks={mediasoup_tidy_checks} -quiet', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -468,13 +501,15 @@ def fuzzer(ctx): ctx.run( f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-fuzzer', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-fuzzer', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -487,7 +522,8 @@ def fuzzer_run_all(ctx): ctx.run( f'LSAN_OPTIONS=verbosity=1:log_threads=1 {BUILD_DIR}/mediasoup-worker-fuzzer -artifact_prefix=fuzzer/reports/ -max_len=1400 fuzzer/new-corpus deps/webrtc-fuzzer-corpora/corpora/stun-corpus deps/webrtc-fuzzer-corpora/corpora/rtp-corpus deps/webrtc-fuzzer-corpora/corpora/rtcp-corpus', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -501,14 +537,16 @@ def docker(ctx): ctx.run( f'{DOCKER} build -f Dockerfile --no-cache --tag mediasoup/docker:latest .', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); else: with ctx.cd(WORKER_DIR): ctx.run( f'{DOCKER} build -f Dockerfile --tag mediasoup/docker:latest .', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -521,7 +559,8 @@ def docker_run(ctx): ctx.run( f'{DOCKER} run --name=mediasoupDocker -it --rm --privileged --cap-add SYS_PTRACE -v {WORKER_DIR}/../:/mediasoup mediasoup/docker:latest', echo=True, - pty=True # NOTE: Needed to enter the terminal of the Docker image. + pty=True, # NOTE: Needed to enter the terminal of the Docker image. + shell=SHELL ); @@ -535,14 +574,16 @@ def docker_alpine(ctx): ctx.run( f'{DOCKER} build -f Dockerfile.alpine --no-cache --tag mediasoup/docker-alpine:latest .', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); else: with ctx.cd(WORKER_DIR): ctx.run( f'{DOCKER} build -f Dockerfile.alpine --tag mediasoup/docker-alpine:latest .', echo=True, - pty=PTY_SUPPORTED + pty=PTY_SUPPORTED, + shell=SHELL ); @@ -555,5 +596,6 @@ def docker_alpine_run(ctx): ctx.run( f'{DOCKER} run --name=mediasoupDockerAlpine -it --rm --privileged --cap-add SYS_PTRACE -v {WORKER_DIR}/../:/mediasoup mediasoup/docker-alpine:latest', echo=True, - pty=True # NOTE: Needed to enter the terminal of the Docker image. + pty=True, # NOTE: Needed to enter the terminal of the Docker image. + shell=SHELL ); From 7b1f82c9d6dce34c05a42ddc1ebb536fc24c2308 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Sat, 25 Nov 2023 11:24:55 +0100 Subject: [PATCH 23/35] Replace make with pip invoke ci playground (#1240) --- .github/workflows/mediasoup-rust.yaml | 4 -- npm-scripts.mjs | 47 +++++++++++--------- worker/Makefile | 63 +++++++++++---------------- worker/build.rs | 48 +++++++++++++++++--- worker/tasks.py | 4 +- 5 files changed, 96 insertions(+), 70 deletions(-) diff --git a/.github/workflows/mediasoup-rust.yaml b/.github/workflows/mediasoup-rust.yaml index f16947b1ef..0778129e84 100644 --- a/.github/workflows/mediasoup-rust.yaml +++ b/.github/workflows/mediasoup-rust.yaml @@ -38,10 +38,6 @@ jobs: ~/.cargo/git key: ${{ matrix.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} - # TODO: Temporal until we decide how to deal with pip invoke requirement. - - name: pip install invoke - run: python3 -m pip install invoke - - name: cargo fmt run: cargo fmt --all -- --check diff --git a/npm-scripts.mjs b/npm-scripts.mjs index 28639cd75c..bd347c83fe 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -9,9 +9,9 @@ import tar from 'tar'; const PKG = JSON.parse(fs.readFileSync('./package.json').toString()); const IS_WINDOWS = os.platform() === 'win32'; const MAYOR_VERSION = PKG.version.split('.')[0]; +const PYTHON = getPython(); const PIP_INVOKE_DIR = 'worker/pip_invoke'; -const INVOKE = `${PIP_INVOKE_DIR}/bin/invoke`; -const INVOKE_VERSION = process.env.INVOKE_VERSION ?? '2.0.0'; +const INVOKE_VERSION = process.env.INVOKE_VERSION ?? '2.2.0'; const FLATBUFFERS_VERSION = '23.3.3'; const WORKER_RELEASE_DIR = 'worker/out/Release'; const WORKER_RELEASE_BIN = IS_WINDOWS ? 'mediasoup-worker.exe' : 'mediasoup-worker'; @@ -161,7 +161,7 @@ async function run() case 'format:worker': { - executeCmd(`${INVOKE} -r worker format`); + executeCmd(`${PYTHON} -m invoke -r worker format`); break; } @@ -293,15 +293,8 @@ async function run() } } -function installInvoke() +function getPython() { - if (fs.existsSync(PIP_INVOKE_DIR)) - { - return; - } - - logInfo('installInvoke()'); - let python = process.env.PYTHON; if (!python) @@ -317,10 +310,22 @@ function installInvoke() } } - // Install `invoke` into custom location, so we don't depend on system-wide + return python; +} + +function installInvoke() +{ + if (fs.existsSync(PIP_INVOKE_DIR)) + { + return; + } + + logInfo('installInvoke()'); + + // Install pip invoke into custom location, so we don't depend on system-wide // installation. executeCmd( - `${python} -m pip install --upgrade --target=${PIP_INVOKE_DIR} invoke==${INVOKE_VERSION}`, /* exitOnError */ true + `${PYTHON} -m pip install --upgrade --target=${PIP_INVOKE_DIR} invoke==${INVOKE_VERSION}`, /* exitOnError */ true ); } @@ -361,7 +366,7 @@ function buildWorker() { logInfo('buildWorker()'); - executeCmd(`${INVOKE} -r worker`); + executeCmd(`${PYTHON} -m invoke -r worker`); } function cleanWorkerArtifacts() @@ -369,11 +374,11 @@ function cleanWorkerArtifacts() logInfo('cleanWorkerArtifacts()'); // Clean build artifacts except `mediasoup-worker`. - executeCmd(`${INVOKE} -r worker clean-build`); + executeCmd(`${PYTHON} -m invoke -r worker clean-build`); // Clean downloaded dependencies. - executeCmd(`${INVOKE} -r worker clean-subprojects`); + executeCmd(`${PYTHON} -m invoke -r worker clean-subprojects`); // Clean PIP/Meson/Ninja. - executeCmd(`${INVOKE} -r worker clean-pip`); + executeCmd(`${PYTHON} -m invoke -r worker clean-pip`); if (IS_WINDOWS) { @@ -395,7 +400,7 @@ function lintWorker() { logInfo('lintWorker()'); - executeCmd(`${INVOKE} -r worker lint`); + executeCmd(`${PYTHON} -m invoke -r worker lint`); } function flatcNode() @@ -403,7 +408,7 @@ function flatcNode() logInfo('flatcNode()'); // Build flatc if needed. - executeCmd(`${INVOKE} -r worker flatc`); + executeCmd(`${PYTHON} -m invoke -r worker flatc`); const buildType = process.env.MEDIASOUP_BUILDTYPE || 'Release'; const extension = IS_WINDOWS ? '.exe' : ''; @@ -428,7 +433,7 @@ function flatcWorker() { logInfo('flatcWorker()'); - executeCmd(`${INVOKE} -r worker flatc`); + executeCmd(`${PYTHON} -m invoke -r worker flatc`); } function testNode() @@ -449,7 +454,7 @@ function testWorker() { logInfo('testWorker()'); - executeCmd(`${INVOKE} -r worker test`); + executeCmd(`${PYTHON} -m invoke -r worker test`); } function installNodeDeps() diff --git a/worker/Makefile b/worker/Makefile index 54a238ee41..9e4b434642 100644 --- a/worker/Makefile +++ b/worker/Makefile @@ -7,8 +7,7 @@ # We need Python 3 here. PYTHON ?= $(shell command -v python3 2> /dev/null || echo python) PIP_INVOKE_DIR = $(shell pwd)/pip_invoke -INVOKE = $(PIP_INVOKE_DIR)/bin/invoke -INVOKE_VERSION ?= 2.0.0 +INVOKE_VERSION ?= 2.2.0 # Instruct Python where to look for invoke module. ifeq ($(OS),Windows_NT) @@ -48,88 +47,76 @@ default: mediasoup-worker invoke: ifeq ($(wildcard $(PIP_INVOKE_DIR)),) - # Install `invoke` into custom location, so we don't depend on system-wide installation. + # Install pip invoke into custom location, so we don't depend on system-wide installation. $(PYTHON) -m pip install --upgrade --target=$(PIP_INVOKE_DIR) invoke==$(INVOKE_VERSION) endif meson-ninja: invoke -ifeq ($(wildcard $(PIP_DIR)),) - # Updated pip and setuptools are needed for meson. - # `--system` is not present everywhere and is only needed as workaround for - # Debian-specific issue (copied from https://github.com/gluster/gstatus/pull/33), - # fallback to command without `--system` if the first one fails. - $(PYTHON) -m pip install --system --target=$(PIP_DIR) pip setuptools || \ - $(PYTHON) -m pip install --target=$(PIP_DIR) pip setuptools || \ - (echo "update of pip or setuptools failed, likely because pip is not installed (if you are on Debian/Ubuntu or derivative please install the python3-pip package)" >&2) - - # Install `meson` and `ninja` using `pip` into custom location, so we don't - # depend on system-wide installation. - $(PYTHON) -m pip install --upgrade --target=$(PIP_DIR) $(PIP_BUILD_BINARIES) meson==$(MESON_VERSION) ninja==$(NINJA_VERSION) -endif + $(PYTHON) -m invoke meson-ninja setup: invoke - $(INVOKE) setup + $(PYTHON) -m invoke setup clean: invoke - $(INVOKE) clean + $(PYTHON) -m invoke clean clean-build: invoke - $(INVOKE) clean-build + $(PYTHON) -m invoke clean-build clean-pip: invoke - $(INVOKE) clean-pip + $(PYTHON) -m invoke clean-pip clean-subprojects: invoke - $(INVOKE) clean-subprojects + $(PYTHON) -m invoke clean-subprojects clean-all: invoke - $(INVOKE) clean-all + $(PYTHON) -m invoke clean-all # It requires the SUBPROJECT environment variable. update-wrap-file: invoke - $(INVOKE) subprojects $(SUBPROJECT) + $(PYTHON) -m invoke subprojects $(SUBPROJECT) mediasoup-worker: invoke - $(INVOKE) mediasoup-worker + $(PYTHON) -m invoke mediasoup-worker libmediasoup-worker: invoke - $(INVOKE) libmediasoup-worker + $(PYTHON) -m invoke libmediasoup-worker flatc: invoke - $(INVOKE) flatc + $(PYTHON) -m invoke flatc xcode: invoke - $(INVOKE) xcode + $(PYTHON) -m invoke xcode lint: invoke - $(INVOKE) lint + $(PYTHON) -m invoke lint format: invoke - $(INVOKE) format + $(PYTHON) -m invoke format test: invoke - $(INVOKE) test + $(PYTHON) -m invoke test test-asan: invoke - $(INVOKE) test-asan + $(PYTHON) -m invoke test-asan tidy: invoke - $(INVOKE) tidy + $(PYTHON) -m invoke tidy fuzzer: invoke - $(INVOKE) fuzzer + $(PYTHON) -m invoke fuzzer fuzzer-run-all: invoke - $(INVOKE) fuzzer-run-all + $(PYTHON) -m invoke fuzzer-run-all docker: invoke - $(INVOKE) docker + $(PYTHON) -m invoke docker docker-run: invoke - $(INVOKE) docker-run + $(PYTHON) -m invoke docker-run docker-alpine: invoke - $(INVOKE) docker-alpine + $(PYTHON) -m invoke docker-alpine docker-alpine-run: invoke - $(INVOKE) docker-alpine-run + $(PYTHON) -m invoke docker-alpine-run diff --git a/worker/build.rs b/worker/build.rs index 32ca69de79..06d25c0ce6 100644 --- a/worker/build.rs +++ b/worker/build.rs @@ -7,8 +7,8 @@ fn main() { return; } - // On Windows Rust always links against release version of MSVC runtime, thus requires Release - // build here. + // On Windows Rust always links against release version of MSVC runtime, thus requires + // Release build here let build_type = if cfg!(all(debug_assertions, not(windows))) { "Debug" } else { @@ -45,7 +45,7 @@ fn main() { ) .expect("Failed to write generated Rust flatbuffers into fbs.rs"); - // Force forward slashes on Windows too so that is plays well with our tasks.py. + // Force forward slashes on Windows too so that is plays well with our tasks.py let mediasoup_out_dir = format!("{}/out", out_dir.replace('\\', "/")); // Add C++ std lib @@ -105,9 +105,44 @@ fn main() { println!("cargo:rustc-link-lib=dylib=c++abi"); } + // Install Python invoke package in custom folder + let pip_invoke_dir = format!("{out_dir}/pip_invoke"); + let invoke_version = "2.2.0"; + let python = env::var("PYTHON").unwrap_or("python3".to_string()); + let mut pythonpath = if env::var("PYTHONPATH").is_ok() { + let original_pythonpath = env::var("PYTHONPATH").unwrap(); + format!("{pip_invoke_dir}:{original_pythonpath}") + } else { + pip_invoke_dir.clone() + }; + + // Force ";" in PYTHONPATH on Windows + if cfg!(target_os = "windows") { + pythonpath = pythonpath.replace(':', ";"); + } + + if !Command::new(&python) + .arg("-m") + .arg("pip") + .arg("install") + .arg("--upgrade") + .arg(format!("--target={pip_invoke_dir}")) + .arg(format!("invoke=={invoke_version}")) + .spawn() + .expect("Failed to start") + .wait() + .expect("Wasn't running") + .success() + { + panic!("Failed to install Python invoke package") + } + // Build - if !Command::new("invoke") + if !Command::new(&python) + .arg("-m") + .arg("invoke") .arg("libmediasoup-worker") + .env("PYTHONPATH", &pythonpath) .env("MEDIASOUP_OUT_DIR", &mediasoup_out_dir) .env("MEDIASOUP_BUILDTYPE", build_type) // Force forward slashes on Windows too, otherwise Meson thinks path is not absolute 🤷 @@ -151,8 +186,11 @@ fn main() { if env::var("KEEP_BUILD_ARTIFACTS") != Ok("1".to_string()) { // Clean - if !Command::new("invoke") + if !Command::new(python) + .arg("-m") + .arg("invoke") .arg("clean-all") + .env("PYTHONPATH", &pythonpath) .env("MEDIASOUP_OUT_DIR", &mediasoup_out_dir) .spawn() .expect("Failed to start") diff --git a/worker/tasks.py b/worker/tasks.py index 4b4e3d4687..25427bf936 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -352,7 +352,7 @@ def xcode(ctx): ); -@task(pre=[meson_ninja]) +@task def lint(ctx): """ Lint source code @@ -376,7 +376,7 @@ def lint(ctx): with ctx.cd(WORKER_DIR): ctx.run( - f'{PYLINT} tasks.py', + f'{PYTHON} -m pylint tasks.py', echo=True, pty=PTY_SUPPORTED, shell=SHELL From 5db18603fde3065a5c36080c9e331fe2a82d0c4f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Sat, 25 Nov 2023 11:29:31 +0100 Subject: [PATCH 24/35] Revert "Replace make with pip invoke ci playground (#1240)" (#1241) This reverts commit 7b1f82c9d6dce34c05a42ddc1ebb536fc24c2308. --- .github/workflows/mediasoup-rust.yaml | 4 ++ npm-scripts.mjs | 47 +++++++++----------- worker/Makefile | 63 ++++++++++++++++----------- worker/build.rs | 48 +++----------------- worker/tasks.py | 4 +- 5 files changed, 70 insertions(+), 96 deletions(-) diff --git a/.github/workflows/mediasoup-rust.yaml b/.github/workflows/mediasoup-rust.yaml index 0778129e84..f16947b1ef 100644 --- a/.github/workflows/mediasoup-rust.yaml +++ b/.github/workflows/mediasoup-rust.yaml @@ -38,6 +38,10 @@ jobs: ~/.cargo/git key: ${{ matrix.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} + # TODO: Temporal until we decide how to deal with pip invoke requirement. + - name: pip install invoke + run: python3 -m pip install invoke + - name: cargo fmt run: cargo fmt --all -- --check diff --git a/npm-scripts.mjs b/npm-scripts.mjs index bd347c83fe..28639cd75c 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -9,9 +9,9 @@ import tar from 'tar'; const PKG = JSON.parse(fs.readFileSync('./package.json').toString()); const IS_WINDOWS = os.platform() === 'win32'; const MAYOR_VERSION = PKG.version.split('.')[0]; -const PYTHON = getPython(); const PIP_INVOKE_DIR = 'worker/pip_invoke'; -const INVOKE_VERSION = process.env.INVOKE_VERSION ?? '2.2.0'; +const INVOKE = `${PIP_INVOKE_DIR}/bin/invoke`; +const INVOKE_VERSION = process.env.INVOKE_VERSION ?? '2.0.0'; const FLATBUFFERS_VERSION = '23.3.3'; const WORKER_RELEASE_DIR = 'worker/out/Release'; const WORKER_RELEASE_BIN = IS_WINDOWS ? 'mediasoup-worker.exe' : 'mediasoup-worker'; @@ -161,7 +161,7 @@ async function run() case 'format:worker': { - executeCmd(`${PYTHON} -m invoke -r worker format`); + executeCmd(`${INVOKE} -r worker format`); break; } @@ -293,8 +293,15 @@ async function run() } } -function getPython() +function installInvoke() { + if (fs.existsSync(PIP_INVOKE_DIR)) + { + return; + } + + logInfo('installInvoke()'); + let python = process.env.PYTHON; if (!python) @@ -310,22 +317,10 @@ function getPython() } } - return python; -} - -function installInvoke() -{ - if (fs.existsSync(PIP_INVOKE_DIR)) - { - return; - } - - logInfo('installInvoke()'); - - // Install pip invoke into custom location, so we don't depend on system-wide + // Install `invoke` into custom location, so we don't depend on system-wide // installation. executeCmd( - `${PYTHON} -m pip install --upgrade --target=${PIP_INVOKE_DIR} invoke==${INVOKE_VERSION}`, /* exitOnError */ true + `${python} -m pip install --upgrade --target=${PIP_INVOKE_DIR} invoke==${INVOKE_VERSION}`, /* exitOnError */ true ); } @@ -366,7 +361,7 @@ function buildWorker() { logInfo('buildWorker()'); - executeCmd(`${PYTHON} -m invoke -r worker`); + executeCmd(`${INVOKE} -r worker`); } function cleanWorkerArtifacts() @@ -374,11 +369,11 @@ function cleanWorkerArtifacts() logInfo('cleanWorkerArtifacts()'); // Clean build artifacts except `mediasoup-worker`. - executeCmd(`${PYTHON} -m invoke -r worker clean-build`); + executeCmd(`${INVOKE} -r worker clean-build`); // Clean downloaded dependencies. - executeCmd(`${PYTHON} -m invoke -r worker clean-subprojects`); + executeCmd(`${INVOKE} -r worker clean-subprojects`); // Clean PIP/Meson/Ninja. - executeCmd(`${PYTHON} -m invoke -r worker clean-pip`); + executeCmd(`${INVOKE} -r worker clean-pip`); if (IS_WINDOWS) { @@ -400,7 +395,7 @@ function lintWorker() { logInfo('lintWorker()'); - executeCmd(`${PYTHON} -m invoke -r worker lint`); + executeCmd(`${INVOKE} -r worker lint`); } function flatcNode() @@ -408,7 +403,7 @@ function flatcNode() logInfo('flatcNode()'); // Build flatc if needed. - executeCmd(`${PYTHON} -m invoke -r worker flatc`); + executeCmd(`${INVOKE} -r worker flatc`); const buildType = process.env.MEDIASOUP_BUILDTYPE || 'Release'; const extension = IS_WINDOWS ? '.exe' : ''; @@ -433,7 +428,7 @@ function flatcWorker() { logInfo('flatcWorker()'); - executeCmd(`${PYTHON} -m invoke -r worker flatc`); + executeCmd(`${INVOKE} -r worker flatc`); } function testNode() @@ -454,7 +449,7 @@ function testWorker() { logInfo('testWorker()'); - executeCmd(`${PYTHON} -m invoke -r worker test`); + executeCmd(`${INVOKE} -r worker test`); } function installNodeDeps() diff --git a/worker/Makefile b/worker/Makefile index 9e4b434642..54a238ee41 100644 --- a/worker/Makefile +++ b/worker/Makefile @@ -7,7 +7,8 @@ # We need Python 3 here. PYTHON ?= $(shell command -v python3 2> /dev/null || echo python) PIP_INVOKE_DIR = $(shell pwd)/pip_invoke -INVOKE_VERSION ?= 2.2.0 +INVOKE = $(PIP_INVOKE_DIR)/bin/invoke +INVOKE_VERSION ?= 2.0.0 # Instruct Python where to look for invoke module. ifeq ($(OS),Windows_NT) @@ -47,76 +48,88 @@ default: mediasoup-worker invoke: ifeq ($(wildcard $(PIP_INVOKE_DIR)),) - # Install pip invoke into custom location, so we don't depend on system-wide installation. + # Install `invoke` into custom location, so we don't depend on system-wide installation. $(PYTHON) -m pip install --upgrade --target=$(PIP_INVOKE_DIR) invoke==$(INVOKE_VERSION) endif meson-ninja: invoke - $(PYTHON) -m invoke meson-ninja +ifeq ($(wildcard $(PIP_DIR)),) + # Updated pip and setuptools are needed for meson. + # `--system` is not present everywhere and is only needed as workaround for + # Debian-specific issue (copied from https://github.com/gluster/gstatus/pull/33), + # fallback to command without `--system` if the first one fails. + $(PYTHON) -m pip install --system --target=$(PIP_DIR) pip setuptools || \ + $(PYTHON) -m pip install --target=$(PIP_DIR) pip setuptools || \ + (echo "update of pip or setuptools failed, likely because pip is not installed (if you are on Debian/Ubuntu or derivative please install the python3-pip package)" >&2) + + # Install `meson` and `ninja` using `pip` into custom location, so we don't + # depend on system-wide installation. + $(PYTHON) -m pip install --upgrade --target=$(PIP_DIR) $(PIP_BUILD_BINARIES) meson==$(MESON_VERSION) ninja==$(NINJA_VERSION) +endif setup: invoke - $(PYTHON) -m invoke setup + $(INVOKE) setup clean: invoke - $(PYTHON) -m invoke clean + $(INVOKE) clean clean-build: invoke - $(PYTHON) -m invoke clean-build + $(INVOKE) clean-build clean-pip: invoke - $(PYTHON) -m invoke clean-pip + $(INVOKE) clean-pip clean-subprojects: invoke - $(PYTHON) -m invoke clean-subprojects + $(INVOKE) clean-subprojects clean-all: invoke - $(PYTHON) -m invoke clean-all + $(INVOKE) clean-all # It requires the SUBPROJECT environment variable. update-wrap-file: invoke - $(PYTHON) -m invoke subprojects $(SUBPROJECT) + $(INVOKE) subprojects $(SUBPROJECT) mediasoup-worker: invoke - $(PYTHON) -m invoke mediasoup-worker + $(INVOKE) mediasoup-worker libmediasoup-worker: invoke - $(PYTHON) -m invoke libmediasoup-worker + $(INVOKE) libmediasoup-worker flatc: invoke - $(PYTHON) -m invoke flatc + $(INVOKE) flatc xcode: invoke - $(PYTHON) -m invoke xcode + $(INVOKE) xcode lint: invoke - $(PYTHON) -m invoke lint + $(INVOKE) lint format: invoke - $(PYTHON) -m invoke format + $(INVOKE) format test: invoke - $(PYTHON) -m invoke test + $(INVOKE) test test-asan: invoke - $(PYTHON) -m invoke test-asan + $(INVOKE) test-asan tidy: invoke - $(PYTHON) -m invoke tidy + $(INVOKE) tidy fuzzer: invoke - $(PYTHON) -m invoke fuzzer + $(INVOKE) fuzzer fuzzer-run-all: invoke - $(PYTHON) -m invoke fuzzer-run-all + $(INVOKE) fuzzer-run-all docker: invoke - $(PYTHON) -m invoke docker + $(INVOKE) docker docker-run: invoke - $(PYTHON) -m invoke docker-run + $(INVOKE) docker-run docker-alpine: invoke - $(PYTHON) -m invoke docker-alpine + $(INVOKE) docker-alpine docker-alpine-run: invoke - $(PYTHON) -m invoke docker-alpine-run + $(INVOKE) docker-alpine-run diff --git a/worker/build.rs b/worker/build.rs index 06d25c0ce6..32ca69de79 100644 --- a/worker/build.rs +++ b/worker/build.rs @@ -7,8 +7,8 @@ fn main() { return; } - // On Windows Rust always links against release version of MSVC runtime, thus requires - // Release build here + // On Windows Rust always links against release version of MSVC runtime, thus requires Release + // build here. let build_type = if cfg!(all(debug_assertions, not(windows))) { "Debug" } else { @@ -45,7 +45,7 @@ fn main() { ) .expect("Failed to write generated Rust flatbuffers into fbs.rs"); - // Force forward slashes on Windows too so that is plays well with our tasks.py + // Force forward slashes on Windows too so that is plays well with our tasks.py. let mediasoup_out_dir = format!("{}/out", out_dir.replace('\\', "/")); // Add C++ std lib @@ -105,44 +105,9 @@ fn main() { println!("cargo:rustc-link-lib=dylib=c++abi"); } - // Install Python invoke package in custom folder - let pip_invoke_dir = format!("{out_dir}/pip_invoke"); - let invoke_version = "2.2.0"; - let python = env::var("PYTHON").unwrap_or("python3".to_string()); - let mut pythonpath = if env::var("PYTHONPATH").is_ok() { - let original_pythonpath = env::var("PYTHONPATH").unwrap(); - format!("{pip_invoke_dir}:{original_pythonpath}") - } else { - pip_invoke_dir.clone() - }; - - // Force ";" in PYTHONPATH on Windows - if cfg!(target_os = "windows") { - pythonpath = pythonpath.replace(':', ";"); - } - - if !Command::new(&python) - .arg("-m") - .arg("pip") - .arg("install") - .arg("--upgrade") - .arg(format!("--target={pip_invoke_dir}")) - .arg(format!("invoke=={invoke_version}")) - .spawn() - .expect("Failed to start") - .wait() - .expect("Wasn't running") - .success() - { - panic!("Failed to install Python invoke package") - } - // Build - if !Command::new(&python) - .arg("-m") - .arg("invoke") + if !Command::new("invoke") .arg("libmediasoup-worker") - .env("PYTHONPATH", &pythonpath) .env("MEDIASOUP_OUT_DIR", &mediasoup_out_dir) .env("MEDIASOUP_BUILDTYPE", build_type) // Force forward slashes on Windows too, otherwise Meson thinks path is not absolute 🤷 @@ -186,11 +151,8 @@ fn main() { if env::var("KEEP_BUILD_ARTIFACTS") != Ok("1".to_string()) { // Clean - if !Command::new(python) - .arg("-m") - .arg("invoke") + if !Command::new("invoke") .arg("clean-all") - .env("PYTHONPATH", &pythonpath) .env("MEDIASOUP_OUT_DIR", &mediasoup_out_dir) .spawn() .expect("Failed to start") diff --git a/worker/tasks.py b/worker/tasks.py index 25427bf936..4b4e3d4687 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -352,7 +352,7 @@ def xcode(ctx): ); -@task +@task(pre=[meson_ninja]) def lint(ctx): """ Lint source code @@ -376,7 +376,7 @@ def lint(ctx): with ctx.cd(WORKER_DIR): ctx.run( - f'{PYTHON} -m pylint tasks.py', + f'{PYLINT} tasks.py', echo=True, pty=PTY_SUPPORTED, shell=SHELL From 7abd48ed6b8e3b4b8b3c11540ce8667746a7f8c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Sat, 25 Nov 2023 11:31:31 +0100 Subject: [PATCH 25/35] Replace make with pip invoke ci playground (#1242) CI is green. --- .github/workflows/mediasoup-rust.yaml | 4 - doc/Building.md | 127 ++++++++++++++++---------- npm-scripts.mjs | 47 +++++----- worker/Makefile | 67 ++++++-------- worker/build.rs | 48 +++++++++- worker/scripts/cpu_cores.sh | 19 ---- worker/scripts/get-pip-invoke.sh | 7 -- worker/tasks.py | 4 +- 8 files changed, 178 insertions(+), 145 deletions(-) delete mode 100755 worker/scripts/cpu_cores.sh delete mode 100755 worker/scripts/get-pip-invoke.sh diff --git a/.github/workflows/mediasoup-rust.yaml b/.github/workflows/mediasoup-rust.yaml index f16947b1ef..0778129e84 100644 --- a/.github/workflows/mediasoup-rust.yaml +++ b/.github/workflows/mediasoup-rust.yaml @@ -38,10 +38,6 @@ jobs: ~/.cargo/git key: ${{ matrix.os }}-cargo-${{ hashFiles('**/Cargo.toml') }} - # TODO: Temporal until we decide how to deal with pip invoke requirement. - - name: pip install invoke - run: python3 -m pip install invoke - - name: cargo fmt run: cargo fmt --all -- --check diff --git a/doc/Building.md b/doc/Building.md index 35b362765a..5e4752e86d 100644 --- a/doc/Building.md +++ b/doc/Building.md @@ -16,11 +16,11 @@ Compiles mediasoup TypeScript code (`lib` folder) JavaScript, places it into the ### `npm run worker:build` -Builds the `mediasoup-worker` binary. It invokes `make`below. +Builds the `mediasoup-worker` binary. It invokes `invoke`below. ### `npm run worker:prebuild` -Creates a prebuilt of `mediasoup-worker` in the `worker/prebuild` folder. +Creates a prebuilt of `mediasoup-worker` binary in the `worker/prebuild` folder. ### `npm run lint` @@ -32,11 +32,23 @@ Validates mediasoup JavaScript files using [ESLint](https://eslint.org). ### `npm run lint:worker` -Validates mediasoup-worker C++ files using [clang-format](https://clang.llvm.org/docs/ClangFormat.html). It invokes `make lint` below. +Validates mediasoup worker C++ files using [clang-format](https://clang.llvm.org/docs/ClangFormat.html). It invokes `invoke lint` below. ### `npm run format:worker` -Rewrites mediasoup-worker C++ files using [clang-format](https://clang.llvm.org/docs/ClangFormat.html). It invokes `make format` below. +Rewrites mediasoup worker C++ files using [clang-format](https://clang.llvm.org/docs/ClangFormat.html). It invokes `invoke format` below. + +### `npm run flatc` + +Runs both `npm run flatc:node` and `npm run flatc:worker`. + +### `npm run flatc:node` + +Compiles [FlatBuffers](https://github.com/google/flatbuffers) `.fbs` files in `worker/fbs` to TypeScript code. + +### `npm run flatc:worker` + +Compiles [FlatBuffers](https://github.com/google/flatbuffers) `.fbs` files in `worker/fbs` to C++ code. ### `npm run test` @@ -48,7 +60,7 @@ Runs [Jest](https://jestjs.io) test units located at `test/` folder. ### `npm run test:worker` -Runs [Catch2](https://github.com/catchorg/Catch2) test units located at `worker/test/` folder. It invokes `make test` below. +Runs [Catch2](https://github.com/catchorg/Catch2) test units located at `worker/test/` folder. It invokes `invoke test` below. ### `npm run coverage:node` @@ -62,115 +74,124 @@ Installs NPM dependencies and updates `package-lock.json`. Installs worker NPM packages needed for local development. +### `npm run release:check` + +Runs linters and tests in Node and C++ code. + ## Rust -The only special feature in Rust case is special environment variable `"KEEP_BUILD_ARTIFACTS", that when set to `1` will allow incremental recompilation of changed C++ sources during hacking on mediasoup. +The only special feature in Rust case is special environment variable "KEEP_BUILD_ARTIFACTS", that when set to "1" will allow incremental recompilation of changed C++ sources during hacking on mediasoup. + It is not necessary for normal usage of mediasoup as a dependency. -## Makefile +## Python Invoke and `tasks.py` file + +mediasoup uses Python [Invoke](https://www.pyinvoke.org/) library for managing and organizing tasks in the `worker` folder (mediasoup worker C++ subproject). `Invoke` is basically a replacemente of `make` + `Makefile` written in Python. mediasoup automatically installs `Invoke` in a local custom path during the installation process (in both Node and Rust) so the user doesn't need to worry about it. + +Tasks are defined in `worker/tasks.py`. For development purposes, developers or contributors can install `Invoke` using `pip3 install invoke` and run tasks below within the `worker` folder. -The `worker` folder contains a `Makefile` for the mediasoup-worker C++ subproject. It includes the following tasks: +See all the tasks by running `invoke --list` within the `worker` folder. -### `make` or `make mediasoup-worker` +### `invoke` (default task) -Alias of ``make mediasoup-worker` below. +Alias of `invoke mediasoup-worker` task below. -### `make meson-ninja` +### `invoke meson-ninja` -Installs `meson` and `ninja`. +Installs `meson` and `ninja` into a local custom path. -### `make clean` +### `invoke clean` Cleans built objects and binaries. -### `make clean-build` +### `invoke clean-build` Cleans built objects and other artifacts, but keeps `mediasoup-worker` binary in place. -### `make clean-pip` +### `invoke clean-pip` Cleans `meson` and `ninja` installed in local prefix with pip. -### `make clean-subprojects` +### `invoke clean-subprojects` Cleans subprojects downloaded with Meson. -### `make clean-all` +### `invoke clean-all` Cleans built objects and binaries, `meson` and `ninja` installed in local prefix with pip and all subprojects downloaded with Meson. -### `make update-wrap-file` +### `invoke update-wrap-file [subproject]` -Update the wrap file of a subproject with Meson. Usage example: +Updates the wrap file of a subproject (those in `worker/subprojects` folder) with Meson. Usage example: ```bash cd worker -make update-wrap-file SUBPROJECT=openssl +invoke update-wrap-file openssl ``` -### `make mediasoup-worker` +### `invoke mediasoup-worker` Builds the `mediasoup-worker` binary at `worker/out/Release/`. If the "MEDIASOUP_MAX_CORES" environment variable is set, the build process will use that number of CPU cores. Otherwise it will auto-detect the number of cores in the machine. -"MEDIASOUP_BUILDTYPE" environment variable controls build types, `Release` and `Debug` are presets optimized for those use cases. -Other build types are possible too, but they are not presets and will require "MESON_ARGS" use to customize build configuration. +"MEDIASOUP_BUILDTYPE" environment variable controls build types, "Release" and "Debug" are presets optimized for those use cases. Other build types are possible too, but they are not presets and will require "MESON_ARGS" use to customize build configuration. + Check the meaning of useful macros in the `worker/include/Logger.hpp` header file if you want to enable tracing or other debug information. Binary is built at `worker/out/MEDIASOUP_BUILDTYPE/build`. -In order to instruct the mediasoup Node.js module to use the `Debug` mediasoup-worker binary, an environment variable must be set before running the Node.js application: +In order to instruct the mediasoup Node.js module to use the "Debug"` `mediasoup-worker` binary, an environment variable must be set before running the Node.js application: ```bash MEDIASOUP_BUILDTYPE=Debug node myapp.js ``` -If the "MEDIASOUP_WORKER_BIN" environment variable is set (it must be an absolute file path), mediasoup will use the it as mediasoup-worker binary and **won't** compile the binary: +If the "MEDIASOUP_WORKER_BIN" environment variable is set (it must be an absolute file path), mediasoup will use the it as `mediasoup-worker` binary and **won't** compile the binary: ```bash MEDIASOUP_WORKER_BIN="/home/xxx/src/foo/mediasoup-worker" node myapp.js ``` -### `make libmediasoup-worker` +### `invoke libmediasoup-worker` Builds the `libmediasoup-worker` static library at `worker/out/Release/`. "MEDIASOUP_MAX_CORES"` and "MEDIASOUP_BUILDTYPE" environment variables from above still apply for static library build. -### `make xcode` +### `invoke xcode` -Builds a Xcode project for the mediasoup-worker subproject. +Builds a Xcode project for the mediasoup worker subproject. -### `make lint` +### `invoke lint` -Validates mediasoup-worker C++ files using [clang-format](https://clang.llvm.org/docs/ClangFormat.html) and rules in `worker/.clang-format`. +Validates mediasoup worker C++ files using [clang-format](https://clang.llvm.org/docs/ClangFormat.html) and rules in `worker/.clang-format`. -### `make format` +### `invoke format` -Rewrites mediasoup-worker C++ files using [clang-format](https://clang.llvm.org/docs/ClangFormat.html). +Rewrites mediasoup worker C++ files using [clang-format](https://clang.llvm.org/docs/ClangFormat.html). -### `make test` +### `invoke test` Builds and runs the `mediasoup-worker-test` binary at `worker/out/Release/` (or at `worker/out/Debug/` if the "MEDIASOUP_BUILDTYPE" environment variable is set to "Debug"), which uses [Catch2](https://github.com/catchorg/Catch2) to run test units located at `worker/test/` folder. -### `make test-asan` +### `invoke test-asan` Run test with Address Sanitizer. -### `make tidy` +### `invoke tidy` Runs [clang-tidy](http://clang.llvm.org/extra/clang-tidy/) and performs C++ code checks following `worker/.clang-tidy` rules. **Requirements:** -* `make clean` and `make` must have been called first. +* `invoke clean` and `invoke mediasoup-worker` must have been called first. * [PyYAML](https://pyyaml.org/) is required. - In OSX install it with `brew install libyaml` and `sudo easy_install-X.Y pyyaml`. "MEDIASOUP_TIDY_CHECKS" environment variable with a comma separated list of checks overrides the checks defined in `.clang-tidy` file. -### `make fuzzer` +### `invoke fuzzer` Builds the `mediasoup-worker-fuzzer` binary (which uses [libFuzzer](http://llvm.org/docs/LibFuzzer.html)) at `worker/out/Release/` (or at `worker/out/Debug/` if the "MEDIASOUP_BUILDTYPE" environment variable is set to "Debug"). @@ -182,11 +203,11 @@ Builds the `mediasoup-worker-fuzzer` binary (which uses [libFuzzer](http://llvm. Read the [Fuzzer](Fuzzer.md) documentation for detailed information. -### `make fuzzer-run-all` +### `invoke fuzzer-run-all` Runs all fuzzer cases. -### `make docker` +### `invoke docker` Builds a Linux Ubuntu Docker image with fuzzer capable clang++ and all dependencies to run mediasoup. @@ -194,21 +215,33 @@ Builds a Linux Ubuntu Docker image with fuzzer capable clang++ and all dependenc ```bash cd worker -./scripts/get-dep.sh clang-fuzzer +scripts/get-dep.sh clang-fuzzer ``` -### `make docker-run` +### `invoke docker-run` -Runs a container of the Ubuntu Docker image created with `make docker`. It automatically executes a `bash` session in the `/mediasoup` directory, which is a Docker volume that points to the real `mediasoup` directory. +Runs a container of the Ubuntu Docker image created with `invoke docker`. It automatically executes a `bash` session in the `/mediasoup` directory, which is a Docker volume that points to the real `mediasoup` directory. -**NOTE:** To install and run mediasoup in the container, previous installation (if any) must be properly cleaned by entering the `worker` directory and running `make clean-all`. +**NOTE:** To install and run mediasoup in the container, previous installation (if any) must be properly cleaned by entering the `worker` directory and running `invoke clean-all`. -### `make docker-alpine` +### `invoke docker-alpine` Builds a Linux Alpine Docker image with all dependencies to run mediasoup. -### `make docker-alpine-run` +### `invoke docker-alpine-run` + +Runs a container of the Alpine Docker image created with `invoke docker-alpine`. It automatically executes an `ash` session in the `/mediasoup` directory, which is a Docker volume that points to the real `mediasoup` directory. + +**NOTE:** To install and run mediasoup in the container, previous installation (if any) must be properly cleaned by entering the `worker` directory and running `invoke clean-all`. + +## Makefile + +The `worker` folder contains a `Makefile` file for the mediasoup worker C++ subproject. It acts as a proxy to the `Invoke` tasks defined in `tasks.py`. The `Makefile` file exists to help developers or contributors that prefer keep using `make` commands. -Runs a container of the Alpine Docker image created with `make docker-alpine`. It automatically executes an `ash` session in the `/mediasoup` directory, which is a Docker volume that points to the real `mediasoup` directory. +All tasks defined in `tasks.py` (see above) are available in `Makefile`. There is only one exception: -**NOTE:** To install and run mediasoup in the container, previous installation (if any) must be properly cleaned by entering the `worker` directory and running `make clean-all`. +- The `update-wrap-file` needs a "SUBPROJECT" environment variable indicating the subproject to update. Usage example: + ```bash + cd worker + make update-wrap-file SUBPROJECT=openssl + ``` diff --git a/npm-scripts.mjs b/npm-scripts.mjs index 28639cd75c..bd347c83fe 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -9,9 +9,9 @@ import tar from 'tar'; const PKG = JSON.parse(fs.readFileSync('./package.json').toString()); const IS_WINDOWS = os.platform() === 'win32'; const MAYOR_VERSION = PKG.version.split('.')[0]; +const PYTHON = getPython(); const PIP_INVOKE_DIR = 'worker/pip_invoke'; -const INVOKE = `${PIP_INVOKE_DIR}/bin/invoke`; -const INVOKE_VERSION = process.env.INVOKE_VERSION ?? '2.0.0'; +const INVOKE_VERSION = process.env.INVOKE_VERSION ?? '2.2.0'; const FLATBUFFERS_VERSION = '23.3.3'; const WORKER_RELEASE_DIR = 'worker/out/Release'; const WORKER_RELEASE_BIN = IS_WINDOWS ? 'mediasoup-worker.exe' : 'mediasoup-worker'; @@ -161,7 +161,7 @@ async function run() case 'format:worker': { - executeCmd(`${INVOKE} -r worker format`); + executeCmd(`${PYTHON} -m invoke -r worker format`); break; } @@ -293,15 +293,8 @@ async function run() } } -function installInvoke() +function getPython() { - if (fs.existsSync(PIP_INVOKE_DIR)) - { - return; - } - - logInfo('installInvoke()'); - let python = process.env.PYTHON; if (!python) @@ -317,10 +310,22 @@ function installInvoke() } } - // Install `invoke` into custom location, so we don't depend on system-wide + return python; +} + +function installInvoke() +{ + if (fs.existsSync(PIP_INVOKE_DIR)) + { + return; + } + + logInfo('installInvoke()'); + + // Install pip invoke into custom location, so we don't depend on system-wide // installation. executeCmd( - `${python} -m pip install --upgrade --target=${PIP_INVOKE_DIR} invoke==${INVOKE_VERSION}`, /* exitOnError */ true + `${PYTHON} -m pip install --upgrade --target=${PIP_INVOKE_DIR} invoke==${INVOKE_VERSION}`, /* exitOnError */ true ); } @@ -361,7 +366,7 @@ function buildWorker() { logInfo('buildWorker()'); - executeCmd(`${INVOKE} -r worker`); + executeCmd(`${PYTHON} -m invoke -r worker`); } function cleanWorkerArtifacts() @@ -369,11 +374,11 @@ function cleanWorkerArtifacts() logInfo('cleanWorkerArtifacts()'); // Clean build artifacts except `mediasoup-worker`. - executeCmd(`${INVOKE} -r worker clean-build`); + executeCmd(`${PYTHON} -m invoke -r worker clean-build`); // Clean downloaded dependencies. - executeCmd(`${INVOKE} -r worker clean-subprojects`); + executeCmd(`${PYTHON} -m invoke -r worker clean-subprojects`); // Clean PIP/Meson/Ninja. - executeCmd(`${INVOKE} -r worker clean-pip`); + executeCmd(`${PYTHON} -m invoke -r worker clean-pip`); if (IS_WINDOWS) { @@ -395,7 +400,7 @@ function lintWorker() { logInfo('lintWorker()'); - executeCmd(`${INVOKE} -r worker lint`); + executeCmd(`${PYTHON} -m invoke -r worker lint`); } function flatcNode() @@ -403,7 +408,7 @@ function flatcNode() logInfo('flatcNode()'); // Build flatc if needed. - executeCmd(`${INVOKE} -r worker flatc`); + executeCmd(`${PYTHON} -m invoke -r worker flatc`); const buildType = process.env.MEDIASOUP_BUILDTYPE || 'Release'; const extension = IS_WINDOWS ? '.exe' : ''; @@ -428,7 +433,7 @@ function flatcWorker() { logInfo('flatcWorker()'); - executeCmd(`${INVOKE} -r worker flatc`); + executeCmd(`${PYTHON} -m invoke -r worker flatc`); } function testNode() @@ -449,7 +454,7 @@ function testWorker() { logInfo('testWorker()'); - executeCmd(`${INVOKE} -r worker test`); + executeCmd(`${PYTHON} -m invoke -r worker test`); } function installNodeDeps() diff --git a/worker/Makefile b/worker/Makefile index 54a238ee41..b46f139346 100644 --- a/worker/Makefile +++ b/worker/Makefile @@ -1,14 +1,12 @@ # # make tasks for mediasoup-worker. # -# NOTE: This Makefile is a proxy to pip invoke commands. +# NOTE: This Makefile is a proxy to pip invoke commands (see tasks.py). # -# We need Python 3 here. PYTHON ?= $(shell command -v python3 2> /dev/null || echo python) PIP_INVOKE_DIR = $(shell pwd)/pip_invoke -INVOKE = $(PIP_INVOKE_DIR)/bin/invoke -INVOKE_VERSION ?= 2.0.0 +INVOKE_VERSION ?= 2.2.0 # Instruct Python where to look for invoke module. ifeq ($(OS),Windows_NT) @@ -48,88 +46,77 @@ default: mediasoup-worker invoke: ifeq ($(wildcard $(PIP_INVOKE_DIR)),) - # Install `invoke` into custom location, so we don't depend on system-wide installation. + # Install pip invoke into custom location, so we don't depend on system-wide + # installation. $(PYTHON) -m pip install --upgrade --target=$(PIP_INVOKE_DIR) invoke==$(INVOKE_VERSION) endif meson-ninja: invoke -ifeq ($(wildcard $(PIP_DIR)),) - # Updated pip and setuptools are needed for meson. - # `--system` is not present everywhere and is only needed as workaround for - # Debian-specific issue (copied from https://github.com/gluster/gstatus/pull/33), - # fallback to command without `--system` if the first one fails. - $(PYTHON) -m pip install --system --target=$(PIP_DIR) pip setuptools || \ - $(PYTHON) -m pip install --target=$(PIP_DIR) pip setuptools || \ - (echo "update of pip or setuptools failed, likely because pip is not installed (if you are on Debian/Ubuntu or derivative please install the python3-pip package)" >&2) - - # Install `meson` and `ninja` using `pip` into custom location, so we don't - # depend on system-wide installation. - $(PYTHON) -m pip install --upgrade --target=$(PIP_DIR) $(PIP_BUILD_BINARIES) meson==$(MESON_VERSION) ninja==$(NINJA_VERSION) -endif + $(PYTHON) -m invoke meson-ninja setup: invoke - $(INVOKE) setup + $(PYTHON) -m invoke setup clean: invoke - $(INVOKE) clean + $(PYTHON) -m invoke clean clean-build: invoke - $(INVOKE) clean-build + $(PYTHON) -m invoke clean-build clean-pip: invoke - $(INVOKE) clean-pip + $(PYTHON) -m invoke clean-pip clean-subprojects: invoke - $(INVOKE) clean-subprojects + $(PYTHON) -m invoke clean-subprojects clean-all: invoke - $(INVOKE) clean-all + $(PYTHON) -m invoke clean-all # It requires the SUBPROJECT environment variable. update-wrap-file: invoke - $(INVOKE) subprojects $(SUBPROJECT) + $(PYTHON) -m invoke subprojects $(SUBPROJECT) mediasoup-worker: invoke - $(INVOKE) mediasoup-worker + $(PYTHON) -m invoke mediasoup-worker libmediasoup-worker: invoke - $(INVOKE) libmediasoup-worker + $(PYTHON) -m invoke libmediasoup-worker flatc: invoke - $(INVOKE) flatc + $(PYTHON) -m invoke flatc xcode: invoke - $(INVOKE) xcode + $(PYTHON) -m invoke xcode lint: invoke - $(INVOKE) lint + $(PYTHON) -m invoke lint format: invoke - $(INVOKE) format + $(PYTHON) -m invoke format test: invoke - $(INVOKE) test + $(PYTHON) -m invoke test test-asan: invoke - $(INVOKE) test-asan + $(PYTHON) -m invoke test-asan tidy: invoke - $(INVOKE) tidy + $(PYTHON) -m invoke tidy fuzzer: invoke - $(INVOKE) fuzzer + $(PYTHON) -m invoke fuzzer fuzzer-run-all: invoke - $(INVOKE) fuzzer-run-all + $(PYTHON) -m invoke fuzzer-run-all docker: invoke - $(INVOKE) docker + $(PYTHON) -m invoke docker docker-run: invoke - $(INVOKE) docker-run + $(PYTHON) -m invoke docker-run docker-alpine: invoke - $(INVOKE) docker-alpine + $(PYTHON) -m invoke docker-alpine docker-alpine-run: invoke - $(INVOKE) docker-alpine-run + $(PYTHON) -m invoke docker-alpine-run diff --git a/worker/build.rs b/worker/build.rs index 32ca69de79..06d25c0ce6 100644 --- a/worker/build.rs +++ b/worker/build.rs @@ -7,8 +7,8 @@ fn main() { return; } - // On Windows Rust always links against release version of MSVC runtime, thus requires Release - // build here. + // On Windows Rust always links against release version of MSVC runtime, thus requires + // Release build here let build_type = if cfg!(all(debug_assertions, not(windows))) { "Debug" } else { @@ -45,7 +45,7 @@ fn main() { ) .expect("Failed to write generated Rust flatbuffers into fbs.rs"); - // Force forward slashes on Windows too so that is plays well with our tasks.py. + // Force forward slashes on Windows too so that is plays well with our tasks.py let mediasoup_out_dir = format!("{}/out", out_dir.replace('\\', "/")); // Add C++ std lib @@ -105,9 +105,44 @@ fn main() { println!("cargo:rustc-link-lib=dylib=c++abi"); } + // Install Python invoke package in custom folder + let pip_invoke_dir = format!("{out_dir}/pip_invoke"); + let invoke_version = "2.2.0"; + let python = env::var("PYTHON").unwrap_or("python3".to_string()); + let mut pythonpath = if env::var("PYTHONPATH").is_ok() { + let original_pythonpath = env::var("PYTHONPATH").unwrap(); + format!("{pip_invoke_dir}:{original_pythonpath}") + } else { + pip_invoke_dir.clone() + }; + + // Force ";" in PYTHONPATH on Windows + if cfg!(target_os = "windows") { + pythonpath = pythonpath.replace(':', ";"); + } + + if !Command::new(&python) + .arg("-m") + .arg("pip") + .arg("install") + .arg("--upgrade") + .arg(format!("--target={pip_invoke_dir}")) + .arg(format!("invoke=={invoke_version}")) + .spawn() + .expect("Failed to start") + .wait() + .expect("Wasn't running") + .success() + { + panic!("Failed to install Python invoke package") + } + // Build - if !Command::new("invoke") + if !Command::new(&python) + .arg("-m") + .arg("invoke") .arg("libmediasoup-worker") + .env("PYTHONPATH", &pythonpath) .env("MEDIASOUP_OUT_DIR", &mediasoup_out_dir) .env("MEDIASOUP_BUILDTYPE", build_type) // Force forward slashes on Windows too, otherwise Meson thinks path is not absolute 🤷 @@ -151,8 +186,11 @@ fn main() { if env::var("KEEP_BUILD_ARTIFACTS") != Ok("1".to_string()) { // Clean - if !Command::new("invoke") + if !Command::new(python) + .arg("-m") + .arg("invoke") .arg("clean-all") + .env("PYTHONPATH", &pythonpath) .env("MEDIASOUP_OUT_DIR", &mediasoup_out_dir) .spawn() .expect("Failed to start") diff --git a/worker/scripts/cpu_cores.sh b/worker/scripts/cpu_cores.sh deleted file mode 100755 index 4419b6f51e..0000000000 --- a/worker/scripts/cpu_cores.sh +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env sh - -set -e - -OS="$(uname -s)" -NUM_CORES= - -case "${OS}" in - Linux*) NUM_CORES=$(nproc);; - Darwin*|FreeBSD) NUM_CORES=$(sysctl -n hw.ncpu);; - MINGW*) NUM_CORES=$NUMBER_OF_PROCESSORS;; - *) NUM_CORES=4;; -esac - -if [ -n "${MEDIASOUP_MAX_CORES}" ]; then - NUM_CORES=$((MEDIASOUP_MAX_CORES>NUM_CORES ? NUM_CORES : MEDIASOUP_MAX_CORES)) -fi - -echo ${NUM_CORES} diff --git a/worker/scripts/get-pip-invoke.sh b/worker/scripts/get-pip-invoke.sh deleted file mode 100755 index b420733e62..0000000000 --- a/worker/scripts/get-pip-invoke.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/usr/bin/env bash - -set -e - -PYTHON=${PYTHON:="python3"} - -${PYTHON} -m pip install invoke diff --git a/worker/tasks.py b/worker/tasks.py index 4b4e3d4687..25427bf936 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -352,7 +352,7 @@ def xcode(ctx): ); -@task(pre=[meson_ninja]) +@task def lint(ctx): """ Lint source code @@ -376,7 +376,7 @@ def lint(ctx): with ctx.cd(WORKER_DIR): ctx.run( - f'{PYLINT} tasks.py', + f'{PYTHON} -m pylint tasks.py', echo=True, pty=PTY_SUPPORTED, shell=SHELL From d56acc201011b211e72b188317d908dd2984a9be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Sat, 25 Nov 2023 11:33:36 +0100 Subject: [PATCH 26/35] Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 115b1b6900..3771b16982 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,11 @@ # Changelog +### NEXT + +* Replace make + Makefile file with Python Invoke library + tasks.py file ([PR #1239](https://github.com/versatica/mediasoup/pull/1239)). + + ### 3.13.5 * Fix RTCP SDES packet size calculation ([PR #1236](https://github.com/versatica/mediasoup/pull/1236) based on PR [PR #1234](https://github.com/versatica/mediasoup/pull/1234) by @ybybwdwd). From e2e6304cf1d9f9eebb975379d88a527a956c2642 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Sat, 25 Nov 2023 12:03:02 +0100 Subject: [PATCH 27/35] CI codeql.yaml: try to fix CI issue (Cannot find package 'node-fetch') directly installing Invoke using pip3 --- .github/workflows/codeql.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/codeql.yaml b/.github/workflows/codeql.yaml index 0055d4ba4e..15c104f781 100644 --- a/.github/workflows/codeql.yaml +++ b/.github/workflows/codeql.yaml @@ -55,9 +55,9 @@ jobs: # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs # queries: security-extended,security-and-quality - # We need to install pip invoke by manually invoking the NPM 'preinstall' script. - - name: npm run preinstall - run: npm run preinstall + # We need to install pip invoke library. + - name: pip3 install invoke + run: pip3 install invoke # Autobuild attempts to build any compiled languages (C/C++, C#, Go, or Java). # If this step fails, then you should remove it and run the build manually (see below) From 90b8e3303640611c0011dbd86ba3b86d257b8109 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Sat, 25 Nov 2023 13:33:17 +0100 Subject: [PATCH 28/35] npm-scripts.mjs: remove old msys stuff - Plus cosmetic changes in CI files --- .github/workflows/mediasoup-worker-prebuild.yaml | 8 ++++---- .github/workflows/mediasoup-worker.yaml | 8 ++++---- npm-scripts.mjs | 8 -------- 3 files changed, 8 insertions(+), 16 deletions(-) diff --git a/.github/workflows/mediasoup-worker-prebuild.yaml b/.github/workflows/mediasoup-worker-prebuild.yaml index 7260e2fca4..ea1491cf72 100644 --- a/.github/workflows/mediasoup-worker-prebuild.yaml +++ b/.github/workflows/mediasoup-worker-prebuild.yaml @@ -54,10 +54,10 @@ jobs: - name: npm ci run: npm ci --ignore-scripts --omit=dev - # However we also need to install pip invoke by manually invoking the NPM - # 'preinstall' script (since `--ignore-scripts` above made it not run). - - name: npm run preinstall - run: npm run preinstall + # However we also need to install pip invoke manually (since + # `--ignore-scripts` prevented invoke from being installed). + - name: pip3 install invoke + run: pip3 install invoke - name: npm run worker:build run: npm run worker:build diff --git a/.github/workflows/mediasoup-worker.yaml b/.github/workflows/mediasoup-worker.yaml index 304e0741c0..104ff9aa89 100644 --- a/.github/workflows/mediasoup-worker.yaml +++ b/.github/workflows/mediasoup-worker.yaml @@ -68,10 +68,10 @@ jobs: - name: npm ci run: npm ci --ignore-scripts --omit=dev - # However we also need to install pip invoke by manually invoking the NPM - # 'preinstall' script (since `--ignore-scripts` above made it not run). - - name: npm run preinstall - run: npm run preinstall + # However we also need to install pip invoke manually (since + # `--ignore-scripts` prevented invoke from being installed). + - name: pip3 install invoke + run: pip3 install invoke # We need to install deps on worker/scripts/package.json. - name: npm run install-worker-dev-tools diff --git a/npm-scripts.mjs b/npm-scripts.mjs index bd347c83fe..82cbb6b3d8 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -379,14 +379,6 @@ function cleanWorkerArtifacts() executeCmd(`${PYTHON} -m invoke -r worker clean-subprojects`); // Clean PIP/Meson/Ninja. executeCmd(`${PYTHON} -m invoke -r worker clean-pip`); - - if (IS_WINDOWS) - { - if (fs.existsSync('worker/out/msys')) - { - executeCmd('rd /s /q worker\\out\\msys'); - } - } } function lintNode() From d5254a4151802d960d77a448014752530d36d23a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Sun, 26 Nov 2023 20:42:26 +0100 Subject: [PATCH 29/35] cosmetic --- npm-scripts.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/npm-scripts.mjs b/npm-scripts.mjs index 82cbb6b3d8..cd0dadc948 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -366,7 +366,7 @@ function buildWorker() { logInfo('buildWorker()'); - executeCmd(`${PYTHON} -m invoke -r worker`); + executeCmd(`${PYTHON} -m invoke -r worker mediasoup-worker`); } function cleanWorkerArtifacts() From c7720f2da4f399a81d1b482cf28d8d604207ff19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Sun, 26 Nov 2023 22:05:44 +0100 Subject: [PATCH 30/35] Allow mediasoup installation under path with whitespaces (fixes #753) # Conflicts: # npm-scripts.mjs --- npm-scripts.mjs | 40 ++++++++--------- worker/build.rs | 2 +- worker/scripts/clang-format.mjs | 4 +- worker/tasks.py | 80 ++++++++++++++++----------------- 4 files changed, 63 insertions(+), 63 deletions(-) diff --git a/npm-scripts.mjs b/npm-scripts.mjs index cd0dadc948..7d814c7a0f 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -161,7 +161,7 @@ async function run() case 'format:worker': { - executeCmd(`${PYTHON} -m invoke -r worker format`); + executeCmd(`"${PYTHON}" -m invoke -r worker format`); break; } @@ -325,7 +325,7 @@ function installInvoke() // Install pip invoke into custom location, so we don't depend on system-wide // installation. executeCmd( - `${PYTHON} -m pip install --upgrade --target=${PIP_INVOKE_DIR} invoke==${INVOKE_VERSION}`, /* exitOnError */ true + `"${PYTHON}" -m pip install --upgrade --target="${PIP_INVOKE_DIR}" invoke==${INVOKE_VERSION}`, /* exitOnError */ true ); } @@ -366,7 +366,7 @@ function buildWorker() { logInfo('buildWorker()'); - executeCmd(`${PYTHON} -m invoke -r worker mediasoup-worker`); + executeCmd(`"${PYTHON}" -m invoke -r worker mediasoup-worker`); } function cleanWorkerArtifacts() @@ -374,11 +374,11 @@ function cleanWorkerArtifacts() logInfo('cleanWorkerArtifacts()'); // Clean build artifacts except `mediasoup-worker`. - executeCmd(`${PYTHON} -m invoke -r worker clean-build`); + executeCmd(`"${PYTHON}" -m invoke -r worker clean-build`); // Clean downloaded dependencies. - executeCmd(`${PYTHON} -m invoke -r worker clean-subprojects`); + executeCmd(`"${PYTHON}" -m invoke -r worker clean-subprojects`); // Clean PIP/Meson/Ninja. - executeCmd(`${PYTHON} -m invoke -r worker clean-pip`); + executeCmd(`"${PYTHON}" -m invoke -r worker clean-pip`); } function lintNode() @@ -392,7 +392,7 @@ function lintWorker() { logInfo('lintWorker()'); - executeCmd(`${PYTHON} -m invoke -r worker lint`); + executeCmd(`"${PYTHON}" -m invoke -r worker lint`); } function flatcNode() @@ -400,24 +400,24 @@ function flatcNode() logInfo('flatcNode()'); // Build flatc if needed. - executeCmd(`${PYTHON} -m invoke -r worker flatc`); + executeCmd(`"${PYTHON}" -m invoke -r worker flatc`); const buildType = process.env.MEDIASOUP_BUILDTYPE || 'Release'; const extension = IS_WINDOWS ? '.exe' : ''; const flatc = path.resolve(path.join( 'worker', 'out', buildType, 'build', 'subprojects', `flatbuffers-${FLATBUFFERS_VERSION}`, `flatc${extension}`)); - const src = path.resolve(path.join('worker', 'fbs', '*.fbs')); const out = path.resolve(path.join('node', 'src')); - const options = '--ts-no-import-ext --gen-object-api'; - const command = `${flatc} --ts ${options} -o ${out} `; - if (IS_WINDOWS) + for (const dirent of fs.readdirSync(path.join('worker', 'fbs'), { withFileTypes: true })) { - executeCmd(`for %f in (${src}) do ${command} %f`); - } - else - { - executeCmd(`for file in ${src}; do ${command} \$\{file\}; done`); + if (!dirent.isFile() || path.parse(dirent.name).ext !== '.fbs') + { + continue; + } + + const filePath = path.resolve(path.join('worker', 'fbs', dirent.name)); + + executeCmd(`"${flatc}" --ts --ts-no-import-ext --gen-object-api -o "${out}" "${filePath}"`); } } @@ -425,7 +425,7 @@ function flatcWorker() { logInfo('flatcWorker()'); - executeCmd(`${PYTHON} -m invoke -r worker flatc`); + executeCmd(`"${PYTHON}" -m invoke -r worker flatc`); } function testNode() @@ -438,7 +438,7 @@ function testNode() } else { - executeCmd(`jest --testPathPattern ${process.env.TEST_FILE}`); + executeCmd(`jest --testPathPattern "${process.env.TEST_FILE}"`); } } @@ -446,7 +446,7 @@ function testWorker() { logInfo('testWorker()'); - executeCmd(`${PYTHON} -m invoke -r worker test`); + executeCmd(`"${PYTHON}" -m invoke -r worker test`); } function installNodeDeps() diff --git a/worker/build.rs b/worker/build.rs index 06d25c0ce6..bc8e1bb371 100644 --- a/worker/build.rs +++ b/worker/build.rs @@ -126,7 +126,7 @@ fn main() { .arg("pip") .arg("install") .arg("--upgrade") - .arg(format!("--target={pip_invoke_dir}")) + .arg(format!("--target=\"{pip_invoke_dir}\"")) .arg(format!("invoke=={invoke_version}")) .spawn() .expect("Failed to start") diff --git a/worker/scripts/clang-format.mjs b/worker/scripts/clang-format.mjs index a7a0f00f4d..13fbd0aaa9 100644 --- a/worker/scripts/clang-format.mjs +++ b/worker/scripts/clang-format.mjs @@ -25,7 +25,7 @@ async function run() case 'lint': { executeCmd( - `${clangFormatNativeBinary} --Werror --dry-run ${workerFiles.join(' ')}` + `"${clangFormatNativeBinary}" --Werror --dry-run ${workerFiles.join(' ')}` ); break; @@ -34,7 +34,7 @@ async function run() case 'format': { executeCmd( - `${clangFormatNativeBinary} --Werror -i ${workerFiles.join(' ')}` + `"${clangFormatNativeBinary}" --Werror -i ${workerFiles.join(' ')}` ); break; diff --git a/worker/tasks.py b/worker/tasks.py index 25427bf936..5a2e906b79 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -103,14 +103,14 @@ def meson_ninja(ctx): # fallback to command without `--system` if the first one fails. try: ctx.run( - f'{PYTHON} -m pip install --system --target={PIP_MESON_NINJA_DIR} pip setuptools', + f'"{PYTHON}" -m pip install --system --upgrade --target="{PIP_MESON_NINJA_DIR}" pip setuptools', echo=True, hide=True, shell=SHELL ); except: ctx.run( - f'{PYTHON} -m pip install --target={PIP_MESON_NINJA_DIR} pip setuptools', + f'"{PYTHON}" -m pip install --upgrade --target="{PIP_MESON_NINJA_DIR}" pip setuptools', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -123,7 +123,7 @@ def meson_ninja(ctx): # Install meson and ninja using pip into our custom location, so we don't # depend on system-wide installation. ctx.run( - f'{PYTHON} -m pip install --upgrade --target={PIP_MESON_NINJA_DIR} {pip_build_binaries} meson=={MESON_VERSION} ninja=={NINJA_VERSION}', + f'"{PYTHON}" -m pip install --upgrade --target="{PIP_MESON_NINJA_DIR}" {pip_build_binaries} meson=={MESON_VERSION} ninja=={NINJA_VERSION}', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -141,7 +141,7 @@ def setup(ctx): try: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', + f'"{MESON}" setup --prefix "{MEDIASOUP_INSTALL_DIR}" --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} "{BUILD_DIR}"', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -149,7 +149,7 @@ def setup(ctx): except: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', + f'"{MESON}" setup --prefix "{MEDIASOUP_INSTALL_DIR}" --bindir "" --libdir "" --buildtype release -Db_ndebug=true -Db_pie=true -Db_staticpic=true {MESON_ARGS} "{BUILD_DIR}"', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -158,7 +158,7 @@ def setup(ctx): try: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', + f'"{MESON}" setup --prefix "{MEDIASOUP_INSTALL_DIR}" --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} "{BUILD_DIR}"', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -166,7 +166,7 @@ def setup(ctx): except: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', + f'"{MESON}" setup --prefix "{MEDIASOUP_INSTALL_DIR}" --bindir "" --libdir "" --buildtype debug -Db_pie=true -Db_staticpic=true {MESON_ARGS} "{BUILD_DIR}"', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -175,7 +175,7 @@ def setup(ctx): try: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} {BUILD_DIR}', + f'"{MESON}" setup --prefix "{MEDIASOUP_INSTALL_DIR}" --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true --reconfigure {MESON_ARGS} "{BUILD_DIR}"', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -183,7 +183,7 @@ def setup(ctx): except: with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --prefix {MEDIASOUP_INSTALL_DIR} --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true {MESON_ARGS} {BUILD_DIR}', + f'"{MESON}" setup --prefix "{MEDIASOUP_INSTALL_DIR}" --bindir "" --libdir "" --buildtype {MEDIASOUP_BUILDTYPE} -Db_ndebug=if-release -Db_pie=true -Db_staticpic=true {MESON_ARGS} "{BUILD_DIR}"', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -235,7 +235,7 @@ def clean_subprojects(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} subprojects purge --include-cache --confirm', + f'"{MESON}" subprojects purge --include-cache --confirm', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -250,7 +250,7 @@ def clean_all(ctx): with ctx.cd(WORKER_DIR): try: ctx.run( - f'{MESON} subprojects purge --include-cache --confirm', + f'"{MESON}" subprojects purge --include-cache --confirm', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -271,7 +271,7 @@ def update_wrap_file(ctx, subproject): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} subprojects update --reset {subproject}', + f'"{MESON}" subprojects update --reset {subproject}', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -285,7 +285,7 @@ def flatc(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} compile -C {BUILD_DIR} flatbuffers-generator', + f'"{MESON}" compile -C "{BUILD_DIR}" flatbuffers-generator', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -303,14 +303,14 @@ def mediasoup_worker(ctx): with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker', + f'"{MESON}" compile -C "{BUILD_DIR}" -j {NUM_CORES} mediasoup-worker', echo=True, pty=PTY_SUPPORTED, shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker', + f'"{MESON}" install -C "{BUILD_DIR}" --no-rebuild --tags mediasoup-worker', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -324,14 +324,14 @@ def libmediasoup_worker(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} libmediasoup-worker', + f'"{MESON}" compile -C "{BUILD_DIR}" -j {NUM_CORES} libmediasoup-worker', echo=True, pty=PTY_SUPPORTED, shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags libmediasoup-worker', + f'"{MESON}" install -C "{BUILD_DIR}" --no-rebuild --tags libmediasoup-worker', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -345,7 +345,7 @@ def xcode(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} setup --buildtype {MEDIASOUP_BUILDTYPE} --backend xcode {MEDIASOUP_OUT_DIR}/xcode', + f'"{MESON}" setup --buildtype {MEDIASOUP_BUILDTYPE} --backend xcode "{MEDIASOUP_OUT_DIR}/xcode"', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -359,7 +359,7 @@ def lint(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{NPM} run lint --prefix scripts/', + f'"{NPM}" run lint --prefix scripts/', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -368,7 +368,7 @@ def lint(ctx): if not os.path.isfile(PYLINT): # Install pylint using pip into our custom location. ctx.run( - f'{PYTHON} -m pip install --upgrade --target={PIP_PYLINT_DIR} pylint=={PYLINT_VERSION}', + f'"{PYTHON}" -m pip install --upgrade --target="{PIP_PYLINT_DIR}" pylint=={PYLINT_VERSION}', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -376,7 +376,7 @@ def lint(ctx): with ctx.cd(WORKER_DIR): ctx.run( - f'{PYTHON} -m pylint tasks.py', + f'"{PYTHON}" -m pylint tasks.py', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -390,7 +390,7 @@ def format(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{NPM} run format --prefix scripts/', + f'"{NPM}" run format --prefix scripts/', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -404,14 +404,14 @@ def test(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-test', + f'"{MESON}" compile -C "{BUILD_DIR}" -j {NUM_CORES} mediasoup-worker-test', echo=True, pty=PTY_SUPPORTED, shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-test', + f'"{MESON}" install -C "{BUILD_DIR}" --no-rebuild --tags mediasoup-worker-test', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -424,21 +424,21 @@ def test(ctx): if os.name == 'nt': with ctx.cd(WORKER_DIR): ctx.run( - f'{BUILD_DIR}/mediasoup-worker-test.exe --invisibles --use-colour=yes {mediasoup_test_tags}', + f'"{BUILD_DIR}/mediasoup-worker-test.exe" --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, pty=PTY_SUPPORTED, shell=SHELL ); else: ctx.run( - f'{LCOV} --directory {WORKER_DIR} --zerocounters', + f'"{LCOV}" --directory "{WORKER_DIR}" --zerocounters', echo=True, pty=PTY_SUPPORTED, shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( - f'{BUILD_DIR}/mediasoup-worker-test --invisibles --use-colour=yes {mediasoup_test_tags}', + f'"{BUILD_DIR}/mediasoup-worker-test" --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -452,14 +452,14 @@ def test_asan(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-test-asan', + f'"{MESON}" compile -C "{BUILD_DIR}" -j {NUM_CORES} mediasoup-worker-test-asan', echo=True, pty=PTY_SUPPORTED, shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-test-asan', + f'"{MESON}" install -C "{BUILD_DIR}" --no-rebuild --tags mediasoup-worker-test-asan', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -469,7 +469,7 @@ def test_asan(ctx): with ctx.cd(WORKER_DIR): ctx.run( - f'ASAN_OPTIONS=detect_leaks=1 {BUILD_DIR}/mediasoup-worker-test-asan --invisibles --use-colour=yes {mediasoup_test_tags}', + f'ASAN_OPTIONS=detect_leaks=1 "{BUILD_DIR}/mediasoup-worker-test-asan" --invisibles --use-colour=yes {mediasoup_test_tags}', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -485,7 +485,7 @@ def tidy(ctx): with ctx.cd(WORKER_DIR): ctx.run( - f'{PYTHON} ./scripts/clang-tidy.py -clang-tidy-binary=./scripts/node_modules/.bin/clang-tidy -clang-apply-replacements-binary=./scripts/node_modules/.bin/clang-apply-replacements -header-filter="(Channel/**/*.hpp|DepLibSRTP.hpp|DepLibUV.hpp|DepLibWebRTC.hpp|DepOpenSSL.hpp|DepUsrSCTP.hpp|LogLevel.hpp|Logger.hpp|MediaSoupError.hpp|RTC/**/*.hpp|Settings.hpp|Utils.hpp|Worker.hpp|common.hpp|handles/**/*.hpp)" -p={BUILD_DIR} -j={NUM_CORES} -checks={mediasoup_tidy_checks} -quiet', + f'"{PYTHON}" ./scripts/clang-tidy.py -clang-tidy-binary=./scripts/node_modules/.bin/clang-tidy -clang-apply-replacements-binary=./scripts/node_modules/.bin/clang-apply-replacements -header-filter="(Channel/**/*.hpp|DepLibSRTP.hpp|DepLibUV.hpp|DepLibWebRTC.hpp|DepOpenSSL.hpp|DepUsrSCTP.hpp|LogLevel.hpp|Logger.hpp|MediaSoupError.hpp|RTC/**/*.hpp|Settings.hpp|Utils.hpp|Worker.hpp|common.hpp|handles/**/*.hpp)" -p="{BUILD_DIR}" -j={NUM_CORES} -checks={mediasoup_tidy_checks} -quiet', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -499,14 +499,14 @@ def fuzzer(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} compile -C {BUILD_DIR} -j {NUM_CORES} mediasoup-worker-fuzzer', + f'"{MESON}" compile -C "{BUILD_DIR}" -j {NUM_CORES} mediasoup-worker-fuzzer', echo=True, pty=PTY_SUPPORTED, shell=SHELL ); with ctx.cd(WORKER_DIR): ctx.run( - f'{MESON} install -C {BUILD_DIR} --no-rebuild --tags mediasoup-worker-fuzzer', + f'"{MESON}" install -C "{BUILD_DIR}" --no-rebuild --tags mediasoup-worker-fuzzer', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -520,7 +520,7 @@ def fuzzer_run_all(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'LSAN_OPTIONS=verbosity=1:log_threads=1 {BUILD_DIR}/mediasoup-worker-fuzzer -artifact_prefix=fuzzer/reports/ -max_len=1400 fuzzer/new-corpus deps/webrtc-fuzzer-corpora/corpora/stun-corpus deps/webrtc-fuzzer-corpora/corpora/rtp-corpus deps/webrtc-fuzzer-corpora/corpora/rtcp-corpus', + f'LSAN_OPTIONS=verbosity=1:log_threads=1 "{BUILD_DIR}/mediasoup-worker-fuzzer" -artifact_prefix=fuzzer/reports/ -max_len=1400 fuzzer/new-corpus deps/webrtc-fuzzer-corpora/corpora/stun-corpus deps/webrtc-fuzzer-corpora/corpora/rtp-corpus deps/webrtc-fuzzer-corpora/corpora/rtcp-corpus', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -535,7 +535,7 @@ def docker(ctx): if os.getenv('DOCKER_NO_CACHE') == 'true': with ctx.cd(WORKER_DIR): ctx.run( - f'{DOCKER} build -f Dockerfile --no-cache --tag mediasoup/docker:latest .', + f'"{DOCKER}" build -f Dockerfile --no-cache --tag mediasoup/docker:latest .', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -543,7 +543,7 @@ def docker(ctx): else: with ctx.cd(WORKER_DIR): ctx.run( - f'{DOCKER} build -f Dockerfile --tag mediasoup/docker:latest .', + f'"{DOCKER}" build -f Dockerfile --tag mediasoup/docker:latest .', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -557,7 +557,7 @@ def docker_run(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{DOCKER} run --name=mediasoupDocker -it --rm --privileged --cap-add SYS_PTRACE -v {WORKER_DIR}/../:/mediasoup mediasoup/docker:latest', + f'"{DOCKER}" run --name=mediasoupDocker -it --rm --privileged --cap-add SYS_PTRACE -v "{WORKER_DIR}/../:/mediasoup" mediasoup/docker:latest', echo=True, pty=True, # NOTE: Needed to enter the terminal of the Docker image. shell=SHELL @@ -572,7 +572,7 @@ def docker_alpine(ctx): if os.getenv('DOCKER_NO_CACHE') == 'true': with ctx.cd(WORKER_DIR): ctx.run( - f'{DOCKER} build -f Dockerfile.alpine --no-cache --tag mediasoup/docker-alpine:latest .', + f'"{DOCKER}" build -f Dockerfile.alpine --no-cache --tag mediasoup/docker-alpine:latest .', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -580,7 +580,7 @@ def docker_alpine(ctx): else: with ctx.cd(WORKER_DIR): ctx.run( - f'{DOCKER} build -f Dockerfile.alpine --tag mediasoup/docker-alpine:latest .', + f'"{DOCKER}" build -f Dockerfile.alpine --tag mediasoup/docker-alpine:latest .', echo=True, pty=PTY_SUPPORTED, shell=SHELL @@ -594,7 +594,7 @@ def docker_alpine_run(ctx): """ with ctx.cd(WORKER_DIR): ctx.run( - f'{DOCKER} run --name=mediasoupDockerAlpine -it --rm --privileged --cap-add SYS_PTRACE -v {WORKER_DIR}/../:/mediasoup mediasoup/docker-alpine:latest', + f'"{DOCKER}" run --name=mediasoupDockerAlpine -it --rm --privileged --cap-add SYS_PTRACE -v "{WORKER_DIR}/../:/mediasoup" mediasoup/docker-alpine:latest', echo=True, pty=True, # NOTE: Needed to enter the terminal of the Docker image. shell=SHELL From 2913eb0285466352d1ce5b1b83ad18a9be458f72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Sun, 26 Nov 2023 23:49:50 +0100 Subject: [PATCH 31/35] Revert change in build.rs since Rust doesn't like \" in format() --- worker/build.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/worker/build.rs b/worker/build.rs index bc8e1bb371..06d25c0ce6 100644 --- a/worker/build.rs +++ b/worker/build.rs @@ -126,7 +126,7 @@ fn main() { .arg("pip") .arg("install") .arg("--upgrade") - .arg(format!("--target=\"{pip_invoke_dir}\"")) + .arg(format!("--target={pip_invoke_dir}")) .arg(format!("invoke=={invoke_version}")) .spawn() .expect("Failed to start") From 89683007eb438875b204eeab36351836e7e47f96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Mon, 27 Nov 2023 10:16:53 +0100 Subject: [PATCH 32/35] Update CHANGELOG --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3771b16982..59b0bfeb9a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ### NEXT -* Replace make + Makefile file with Python Invoke library + tasks.py file ([PR #1239](https://github.com/versatica/mediasoup/pull/1239)). +* Replace make + Makefile with Python Invoke library + tasks.py (also fix installation under path with whitespaces) ([PR #1239](https://github.com/versatica/mediasoup/pull/1239)). ### 3.13.5 From 42415cae1426f6e5a50082ae738b0bc6b99ec30b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Mon, 27 Nov 2023 11:59:33 +0100 Subject: [PATCH 33/35] Remove useless npm scripts in main package.json and make mediasoup-worker.yaml CI directly use Invoke commands instead of npm commands --- .github/workflows/mediasoup-worker.yaml | 21 ++++++++++----------- doc/Building.md | 14 ++++++-------- npm-scripts.mjs | 14 -------------- package.json | 2 -- 4 files changed, 16 insertions(+), 35 deletions(-) diff --git a/.github/workflows/mediasoup-worker.yaml b/.github/workflows/mediasoup-worker.yaml index 104ff9aa89..4671e169f2 100644 --- a/.github/workflows/mediasoup-worker.yaml +++ b/.github/workflows/mediasoup-worker.yaml @@ -68,26 +68,25 @@ jobs: - name: npm ci run: npm ci --ignore-scripts --omit=dev - # However we also need to install pip invoke manually (since - # `--ignore-scripts` prevented invoke from being installed). + # We need to install pip invoke manually. - name: pip3 install invoke run: pip3 install invoke - # We need to install deps on worker/scripts/package.json. - - name: npm run install-worker-dev-tools - run: npm run install-worker-dev-tools + # We need to install npm deps of worker/scripts/package.json. + - name: npm ci --prefix worker/scripts + run: npm ci --prefix worker/scripts # TODO: Maybe fix this one day. if: runner.os != 'Windows' - - name: npm run lint:worker - run: npm run lint:worker + - name: invoke -r worker lint + run: invoke -r worker lint # TODO: Maybe fix this one day. if: runner.os != 'Windows' - - name: npm run worker:build - run: npm run worker:build + - name: invoke -r worker mediasoup-worker + run: invoke -r worker mediasoup-worker - - name: npm run test:worker - run: npm run test:worker + - name: invoke -r worker test + run: invoke -r worker test # TODO: Maybe fix this one day. if: runner.os != 'Windows' diff --git a/doc/Building.md b/doc/Building.md index 5e4752e86d..7408111f9e 100644 --- a/doc/Building.md +++ b/doc/Building.md @@ -66,14 +66,6 @@ Runs [Catch2](https://github.com/catchorg/Catch2) test units located at `worker/ Same as `npm run test:node` but it also opens a browser window with JavaScript coverage results. -### `npm run install-deps:node` - -Installs NPM dependencies and updates `package-lock.json`. - -### `npm run install-worker-dev-tools` - -Installs worker NPM packages needed for local development. - ### `npm run release:check` Runs linters and tests in Node and C++ code. @@ -92,6 +84,12 @@ Tasks are defined in `worker/tasks.py`. For development purposes, developers or See all the tasks by running `invoke --list` within the `worker` folder. +*NOTE:* For some of these tasks to work, npm dependencies of `worker/scripts/package.json` must be installed: + +```bash +npm ci --prefix worker/scripts +``` + ### `invoke` (default task) Alias of `invoke mediasoup-worker` task below. diff --git a/npm-scripts.mjs b/npm-scripts.mjs index 7d814c7a0f..855c473644 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -204,20 +204,6 @@ async function run() break; } - case 'install-deps:node': - { - installNodeDeps(); - - break; - } - - case 'install-worker-dev-tools': - { - executeCmd('npm ci --prefix worker/scripts'); - - break; - } - case 'release:check': { checkRelease(); diff --git a/package.json b/package.json index 83c681a6e6..c21cb41714 100644 --- a/package.json +++ b/package.json @@ -68,8 +68,6 @@ "test:node": "node npm-scripts.mjs test:node", "test:worker": "node npm-scripts.mjs test:worker", "coverage:node": "node npm-scripts.mjs coverage:node", - "install-deps:node": "node npm-scripts.mjs install-deps:node", - "install-worker-dev-tools": "node npm-scripts.mjs install-worker-dev-tools", "release:check": "node npm-scripts.mjs release:check", "release": "node npm-scripts.mjs release", "release:upload-mac-arm-prebuilt-worker": "node npm-scripts.mjs release:upload-mac-arm-prebuilt-worker" From 482562ce7bd550e02d9061553277404491f0ca34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Mon, 27 Nov 2023 12:06:59 +0100 Subject: [PATCH 34/35] Forgot to remove useless npm ci in mediasoup-worker.yaml --- .github/workflows/mediasoup-worker.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/mediasoup-worker.yaml b/.github/workflows/mediasoup-worker.yaml index 4671e169f2..ac0cb4d042 100644 --- a/.github/workflows/mediasoup-worker.yaml +++ b/.github/workflows/mediasoup-worker.yaml @@ -64,10 +64,6 @@ jobs: restore-keys: | ${{ matrix.build.os }}-node-${{matrix.build.cc}}- - # We need to install some NPM production deps for npm-scripts.mjs to work. - - name: npm ci - run: npm ci --ignore-scripts --omit=dev - # We need to install pip invoke manually. - name: pip3 install invoke run: pip3 install invoke From 79921086bd4224c7bd6ce60894a10dadbe1231f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?I=C3=B1aki=20Baz=20Castillo?= Date: Mon, 27 Nov 2023 12:56:54 +0100 Subject: [PATCH 35/35] Fix PYTHONPATH usage --- npm-scripts.mjs | 15 +++++++++++---- worker/Makefile | 2 +- worker/tasks.py | 9 +++++---- 3 files changed, 17 insertions(+), 9 deletions(-) diff --git a/npm-scripts.mjs b/npm-scripts.mjs index 855c473644..80ebc35c3e 100644 --- a/npm-scripts.mjs +++ b/npm-scripts.mjs @@ -10,7 +10,7 @@ const PKG = JSON.parse(fs.readFileSync('./package.json').toString()); const IS_WINDOWS = os.platform() === 'win32'; const MAYOR_VERSION = PKG.version.split('.')[0]; const PYTHON = getPython(); -const PIP_INVOKE_DIR = 'worker/pip_invoke'; +const PIP_INVOKE_DIR = path.resolve('worker/pip_invoke'); const INVOKE_VERSION = process.env.INVOKE_VERSION ?? '2.2.0'; const FLATBUFFERS_VERSION = '23.3.3'; const WORKER_RELEASE_DIR = 'worker/out/Release'; @@ -26,13 +26,20 @@ const task = process.argv.slice(2).join(' '); // PYTHONPATH env must be updated now so all invoke calls below will find the // pip invoke module. -if (IS_WINDOWS) +if (process.env.PYTHONPATH) { - process.env.PYTHONPATH = `${PIP_INVOKE_DIR};${process.env.PYTHONPATH}`; + if (IS_WINDOWS) + { + process.env.PYTHONPATH = `${PIP_INVOKE_DIR};${process.env.PYTHONPATH}`; + } + else + { + process.env.PYTHONPATH = `${PIP_INVOKE_DIR}:${process.env.PYTHONPATH}`; + } } else { - process.env.PYTHONPATH = `${PIP_INVOKE_DIR}:${process.env.PYTHONPATH}`; + process.env.PYTHONPATH = PIP_INVOKE_DIR; } run(); diff --git a/worker/Makefile b/worker/Makefile index b46f139346..7a0492128c 100644 --- a/worker/Makefile +++ b/worker/Makefile @@ -48,7 +48,7 @@ invoke: ifeq ($(wildcard $(PIP_INVOKE_DIR)),) # Install pip invoke into custom location, so we don't depend on system-wide # installation. - $(PYTHON) -m pip install --upgrade --target=$(PIP_INVOKE_DIR) invoke==$(INVOKE_VERSION) + $(PYTHON) -m pip install --upgrade --target="$(PIP_INVOKE_DIR)" invoke==$(INVOKE_VERSION) endif meson-ninja: invoke diff --git a/worker/tasks.py b/worker/tasks.py index 5a2e906b79..6e471b2989 100644 --- a/worker/tasks.py +++ b/worker/tasks.py @@ -31,6 +31,8 @@ MEDIASOUP_OUT_DIR = os.getenv('MEDIASOUP_OUT_DIR') or f'{WORKER_DIR}/out'; MEDIASOUP_INSTALL_DIR = os.getenv('MEDIASOUP_INSTALL_DIR') or f'{MEDIASOUP_OUT_DIR}/{MEDIASOUP_BUILDTYPE}'; BUILD_DIR = os.getenv('BUILD_DIR') or f'{MEDIASOUP_INSTALL_DIR}/build'; +# Custom pip folder for invoke package. +PIP_INVOKE_DIR = f'{MEDIASOUP_OUT_DIR}/pip_invoke'; # Custom pip folder for meson and ninja packages. PIP_MESON_NINJA_DIR = f'{MEDIASOUP_OUT_DIR}/pip_meson_ninja'; # Custom pip folder for pylint package. @@ -57,7 +59,6 @@ # https://github.com/ninja-build/ninja/issues/2211 # https://github.com/ninja-build/ninja/issues/2212 NINJA_VERSION = os.getenv('NINJA_VERSION') or '1.10.2.4'; -PYLINT = f'{PIP_PYLINT_DIR}/bin/pylint'; PYLINT_VERSION = os.getenv('PYLINT_VERSION') or '3.0.2'; NPM = os.getenv('NPM') or 'npm'; LCOV = f'{WORKER_DIR}/deps/lcov/bin/lcov'; @@ -84,9 +85,9 @@ # NOTE: On Windows we must use ; instead of : to separate paths. PYTHONPATH = os.getenv('PYTHONPATH') or ''; if os.name == 'nt': - os.environ['PYTHONPATH'] = f'{PIP_MESON_NINJA_DIR};{PIP_PYLINT_DIR};{PYTHONPATH}'; + os.environ['PYTHONPATH'] = f'{PIP_INVOKE_DIR};{PIP_MESON_NINJA_DIR};{PIP_PYLINT_DIR};{PYTHONPATH}'; else: - os.environ['PYTHONPATH'] = f'{PIP_MESON_NINJA_DIR}:{PIP_PYLINT_DIR}:{PYTHONPATH}'; + os.environ['PYTHONPATH'] = f'{PIP_INVOKE_DIR}:{PIP_MESON_NINJA_DIR}:{PIP_PYLINT_DIR}:{PYTHONPATH}'; @task @@ -365,7 +366,7 @@ def lint(ctx): shell=SHELL ); - if not os.path.isfile(PYLINT): + if not os.path.isdir(PIP_PYLINT_DIR): # Install pylint using pip into our custom location. ctx.run( f'"{PYTHON}" -m pip install --upgrade --target="{PIP_PYLINT_DIR}" pylint=={PYLINT_VERSION}',