From 665dff7068b7e167b30c52977f243cd603e04078 Mon Sep 17 00:00:00 2001 From: anutosh491 Date: Sun, 22 Dec 2024 10:42:52 +0530 Subject: [PATCH 1/6] Add Initial Support for xeus-cpp-lite --- .github/workflows/deploy-github-page.yml | 4 +++ .github/workflows/main.yml | 4 +++ CMakeLists.txt | 36 +++++++++++++++++++++--- README.md | 12 +++++++- include/xeus-cpp/xinterpreter_wasm.hpp | 3 ++ src/xinterpreter.cpp | 11 ++++++-- src/xinterpreter_wasm.cpp | 8 +++++- 7 files changed, 70 insertions(+), 8 deletions(-) diff --git a/.github/workflows/deploy-github-page.yml b/.github/workflows/deploy-github-page.yml index fa0e7442..0fae9049 100644 --- a/.github/workflows/deploy-github-page.yml +++ b/.github/workflows/deploy-github-page.yml @@ -59,6 +59,7 @@ jobs: echo "PREFIX=$PREFIX" >> $GITHUB_ENV export CMAKE_PREFIX_PATH=$PREFIX export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX + export SYSROOT_PATH=$HOME/emsdk/upstream/emscripten/cache/sysroot emcmake cmake \ -DCMAKE_BUILD_TYPE=Release \ @@ -66,6 +67,7 @@ jobs: -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON \ -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON \ + -DSYSROOT_PATH=$SYSROOT_PATH \ .. emmake make -j ${{ env.ncpus }} install @@ -76,6 +78,8 @@ jobs: micromamba activate xeus-lite-host python -m pip install jupyterlite-xeus jupyter lite build --XeusAddon.prefix=${{ env.PREFIX }} --output-dir dist + #cp xcpp.data dist/extensions/@jupyterlite/xeus/static + #cp $PREFIX/lib/libclangCppInterOp.so dist/extensions/@jupyterlite/xeus/static mkdir -p dist/files mv notebooks dist/files mv README.md dist/files diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index c1724e4e..14713470 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -257,6 +257,7 @@ jobs: echo "PREFIX=$PREFIX" >> $GITHUB_ENV export CMAKE_PREFIX_PATH=$PREFIX export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX + export SYSROOT_PATH=$HOME/emsdk/upstream/emscripten/cache/sysroot emcmake cmake \ -DCMAKE_BUILD_TYPE=Release \ @@ -264,6 +265,7 @@ jobs: -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON \ -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON \ + -DSYSROOT_PATH=$SYSROOT_PATH \ .. emmake make -j ${{ env.ncpus }} install @@ -274,6 +276,8 @@ jobs: micromamba activate xeus-lite-host python -m pip install jupyterlite-xeus jupyter lite build --XeusAddon.prefix=${{ env.PREFIX }} + #cp xcpp.data _output/extensions/@jupyterlite/xeus/static + #cp $PREFIX/lib/libclangCppInterOp.so _output/extensions/@jupyterlite/xeus/static - name: Setup tmate session if: ${{ failure() && runner.debug }} diff --git a/CMakeLists.txt b/CMakeLists.txt index 31ab854a..f0bc1622 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -84,7 +84,7 @@ if(EMSCRIPTEN) set(XEUS_CPP_USE_SHARED_XEUS_CPP OFF) set(XEUS_CPP_BUILD_TESTS OFF) # ENV (https://github.com/emscripten-core/emscripten/commit/6d9681ad04f60b41ef6345ab06c29bbc9eeb84e0) - set(EMSCRIPTEN_FEATURES "${EMSCRIPTEN_FEATURES} -s \"EXTRA_EXPORTED_RUNTIME_METHODS=[ENV']\"") + set(EMSCRIPTEN_FEATURES "${EMSCRIPTEN_FEATURES} -s \"EXPORTED_RUNTIME_METHODS=[ENV']\"") endif() # Dependencies @@ -152,8 +152,22 @@ function(configure_kernel kernel) endfunction() message("Configure kernels: ...") -configure_kernel("/share/jupyter/kernels/xcpp17/") -configure_kernel("/share/jupyter/kernels/xcpp20/") +if(NOT EMSCRIPTEN) + configure_kernel("/share/jupyter/kernels/xcpp17/") + configure_kernel("/share/jupyter/kernels/xcpp20/") +else() + # TODO: Currently jupyterlite-xeus and xeus-lite do not provide + # methods to fetch information from the arguments present in the + # generated emscripten kernel. + # The following needs to be done here : + # 1) We need to configure the kernel properly + # Check issue https://github.com/compiler-research/xeus-cpp/issues/185. + # 2) Once the above is done we need to add support in jupyterlite-xeus & xeus-lite + # to be able to deal with arguments present in kernel.json + # 3) Finally we should fetch the C++ version from the kernel.json file and + # be able to pass it to our wasm interpreter rather than forcing a version. + configure_kernel("/share/jupyter/kernels/xcpp20/") +endif() # Source files # ============ @@ -401,8 +415,22 @@ if(EMSCRIPTEN) xeus_cpp_set_kernel_options(xcpp) xeus_wasm_compile_options(xcpp) xeus_wasm_link_options(xcpp "web,worker") + target_link_options(xcpp PUBLIC + -sEXPORTED_RUNTIME_METHODS=FS,PATH,ERRNO_CODES + # add sysroot location here + --preload-file ${SYSROOT_PATH}/include@/include + ) + # TODO: Emscripten supports preloading files just once before it generates + # the xcpp.data file (containing the binary representation of the file(s) we + # want to include in our application). + # Hence although we are adding supporting for Standard Headers, Libraries etc + # through emscripten's sysroot for now, we need to do the following: + # 1) Enable CppInterOp to provide us with a resource dir. + # 2) If the above cannot be done, we can use the resource dir provided + # by llvm on emscripten-forge but would involve adding a dependency. + # 3) Shift the resource dir and the sysroot to a common location. + # 4) Preload everything required together. endif() - # Tests # ===== diff --git a/README.md b/README.md index a25a0cac..b997360e 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,7 @@ pushd build export PREFIX=$MAMBA_ROOT_PREFIX/envs/xeus-cpp-wasm-host export CMAKE_PREFIX_PATH=$PREFIX export CMAKE_SYSTEM_PREFIX_PATH=$PREFIX +export SYSROOT_PATH=$HOME/emsdk/upstream/emscripten/cache/sysroot emcmake cmake \ -DCMAKE_BUILD_TYPE=Release \ @@ -94,6 +95,7 @@ emcmake cmake \ -DCMAKE_INSTALL_PREFIX=$PREFIX \ -DXEUS_CPP_EMSCRIPTEN_WASM_BUILD=ON \ -DCMAKE_FIND_ROOT_PATH_MODE_PACKAGE=ON \ + -DSYSROOT_PATH=$SYSROOT_PATH \ .. emmake make install ``` @@ -105,12 +107,20 @@ micromamba activate xeus-lite-host python -m pip install jupyterlite-xeus jupyter lite build --XeusAddon.prefix=$PREFIX ``` + +We now need to shift necessary files like `xcpp.data` which contains the binary representation of the file(s) +we want to include in our application. As of now this would contain all important files like Standard Headers, +Libraries etc coming out of emscripten's sysroot. Assuming we are still inside build we should do the following +```bash +cp xcpp.data _output/extensions/@jupyterlite/xeus/static +cp $PREFIX/lib/libclangCppInterOp.so _output/extensions/@jupyterlite/xeus/static +``` + Once the Jupyter Lite site has built you can test the website locally by executing ```bash jupyter lite serve --XeusAddon.prefix=$PREFIX ``` - ## Trying it online To try out xeus-cpp interactively in your web browser, just click on the binder link: diff --git a/include/xeus-cpp/xinterpreter_wasm.hpp b/include/xeus-cpp/xinterpreter_wasm.hpp index abb7614e..2c45b1a1 100644 --- a/include/xeus-cpp/xinterpreter_wasm.hpp +++ b/include/xeus-cpp/xinterpreter_wasm.hpp @@ -22,6 +22,9 @@ namespace xcpp wasm_interpreter(); virtual ~wasm_interpreter() = default; + private: + + static std::vector create_args(); }; } diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp index f099b688..1891d551 100644 --- a/src/xinterpreter.cpp +++ b/src/xinterpreter.cpp @@ -28,6 +28,7 @@ using Args = std::vector; void* createInterpreter(const Args &ExtraArgs = {}) { Args ClangArgs = {/*"-xc++"*/"-v"}; // ? {"-Xclang", "-emit-llvm-only", "-Xclang", "-diagnostic-log-file", "-Xclang", "-", "-xc++"}; +#ifndef EMSCRIPTEN if (std::find_if(ExtraArgs.begin(), ExtraArgs.end(), [](const std::string& s) { return s == "-resource-dir";}) == ExtraArgs.end()) { std::string resource_dir = Cpp::DetectResourceDir(); @@ -42,6 +43,7 @@ void* createInterpreter(const Args &ExtraArgs = {}) { ClangArgs.push_back("-isystem"); ClangArgs.push_back(CxxInclude.c_str()); } +#endif ClangArgs.insert(ClangArgs.end(), ExtraArgs.begin(), ExtraArgs.end()); // FIXME: We should process the kernel input options and conditionally pass // the gpu args here. @@ -73,6 +75,7 @@ namespace xcpp static std::string get_stdopt() { // We need to find what's the C++ version the interpreter runs with. +#ifndef EMSCRIPTEN const char* code = R"( int __get_cxx_version () { #if __cplusplus > 202302L @@ -93,12 +96,14 @@ int __get_cxx_version () { } __get_cxx_version () )"; - auto cxx_version = Cpp::Evaluate(code); return std::to_string(cxx_version); +#else + constexpr int cxx_version = 20; + return std::to_string(cxx_version); +#endif } - interpreter::interpreter(int argc, const char* const* argv) : xmagics() , p_cout_strbuf(nullptr) @@ -357,7 +362,9 @@ __get_cxx_version () void interpreter::init_includes() { +#ifndef EMSCRIPTEN Cpp::AddIncludePath((xeus::prefix_path() + "/include/").c_str()); +#endif } void interpreter::init_preamble() diff --git a/src/xinterpreter_wasm.cpp b/src/xinterpreter_wasm.cpp index 7746c7b6..c3980773 100644 --- a/src/xinterpreter_wasm.cpp +++ b/src/xinterpreter_wasm.cpp @@ -17,7 +17,13 @@ namespace xcpp { wasm_interpreter::wasm_interpreter() - : interpreter(0, nullptr) + : interpreter(create_args().size(), create_args().data()) { } + + std::vector wasm_interpreter::create_args() + { + static std::vector Args = {"-Xclang", "-std=c++20"}; + return Args; + } } From 7b92a8f16db04096461388686b0b352a2c588cb4 Mon Sep 17 00:00:00 2001 From: anutosh491 Date: Sun, 22 Dec 2024 11:35:08 +0530 Subject: [PATCH 2/6] install xcpp.data too --- CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index f0bc1622..061a373a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -520,6 +520,7 @@ if(EMSCRIPTEN) install(FILES "$/xcpp.js" "$/xcpp.wasm" + "$/xcpp.data" DESTINATION ${CMAKE_INSTALL_BINDIR}) endif () From 83f6d17072e54f3d9061e57dfb3dcb888310ca41 Mon Sep 17 00:00:00 2001 From: anutosh491 Date: Sun, 22 Dec 2024 16:54:17 +0530 Subject: [PATCH 3/6] uncomment needed comments --- .github/workflows/deploy-github-page.yml | 4 ++-- .github/workflows/main.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy-github-page.yml b/.github/workflows/deploy-github-page.yml index 0fae9049..b0f7a049 100644 --- a/.github/workflows/deploy-github-page.yml +++ b/.github/workflows/deploy-github-page.yml @@ -78,8 +78,8 @@ jobs: micromamba activate xeus-lite-host python -m pip install jupyterlite-xeus jupyter lite build --XeusAddon.prefix=${{ env.PREFIX }} --output-dir dist - #cp xcpp.data dist/extensions/@jupyterlite/xeus/static - #cp $PREFIX/lib/libclangCppInterOp.so dist/extensions/@jupyterlite/xeus/static + cp xcpp.data dist/extensions/@jupyterlite/xeus/static + cp $PREFIX/lib/libclangCppInterOp.so dist/extensions/@jupyterlite/xeus/static mkdir -p dist/files mv notebooks dist/files mv README.md dist/files diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 14713470..d663307f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -276,8 +276,8 @@ jobs: micromamba activate xeus-lite-host python -m pip install jupyterlite-xeus jupyter lite build --XeusAddon.prefix=${{ env.PREFIX }} - #cp xcpp.data _output/extensions/@jupyterlite/xeus/static - #cp $PREFIX/lib/libclangCppInterOp.so _output/extensions/@jupyterlite/xeus/static + cp xcpp.data _output/extensions/@jupyterlite/xeus/static + cp $PREFIX/lib/libclangCppInterOp.so _output/extensions/@jupyterlite/xeus/static - name: Setup tmate session if: ${{ failure() && runner.debug }} From ce35626897d4864b5b4b04e086a0a45500f4e6ab Mon Sep 17 00:00:00 2001 From: anutosh491 Date: Mon, 23 Dec 2024 01:33:53 +0530 Subject: [PATCH 4/6] Address reviews --- CMakeLists.txt | 12 +++++++----- include/xeus-cpp/xinterpreter_wasm.hpp | 3 --- src/xinterpreter.cpp | 15 ++++++--------- src/xinterpreter_wasm.cpp | 8 +------- 4 files changed, 14 insertions(+), 24 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 061a373a..a9a08845 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -152,10 +152,7 @@ function(configure_kernel kernel) endfunction() message("Configure kernels: ...") -if(NOT EMSCRIPTEN) - configure_kernel("/share/jupyter/kernels/xcpp17/") - configure_kernel("/share/jupyter/kernels/xcpp20/") -else() +if(EMSCRIPTEN) # TODO: Currently jupyterlite-xeus and xeus-lite do not provide # methods to fetch information from the arguments present in the # generated emscripten kernel. @@ -167,6 +164,9 @@ else() # 3) Finally we should fetch the C++ version from the kernel.json file and # be able to pass it to our wasm interpreter rather than forcing a version. configure_kernel("/share/jupyter/kernels/xcpp20/") +else() + configure_kernel("/share/jupyter/kernels/xcpp17/") + configure_kernel("/share/jupyter/kernels/xcpp20/") endif() # Source files @@ -415,6 +415,8 @@ if(EMSCRIPTEN) xeus_cpp_set_kernel_options(xcpp) xeus_wasm_compile_options(xcpp) xeus_wasm_link_options(xcpp "web,worker") + # TODO: Remove the exported runtime methods + # after the next xeus release. target_link_options(xcpp PUBLIC -sEXPORTED_RUNTIME_METHODS=FS,PATH,ERRNO_CODES # add sysroot location here @@ -423,7 +425,7 @@ if(EMSCRIPTEN) # TODO: Emscripten supports preloading files just once before it generates # the xcpp.data file (containing the binary representation of the file(s) we # want to include in our application). - # Hence although we are adding supporting for Standard Headers, Libraries etc + # Hence although we are adding support for Standard Headers, Libraries etc # through emscripten's sysroot for now, we need to do the following: # 1) Enable CppInterOp to provide us with a resource dir. # 2) If the above cannot be done, we can use the resource dir provided diff --git a/include/xeus-cpp/xinterpreter_wasm.hpp b/include/xeus-cpp/xinterpreter_wasm.hpp index 2c45b1a1..abb7614e 100644 --- a/include/xeus-cpp/xinterpreter_wasm.hpp +++ b/include/xeus-cpp/xinterpreter_wasm.hpp @@ -22,9 +22,6 @@ namespace xcpp wasm_interpreter(); virtual ~wasm_interpreter() = default; - private: - - static std::vector create_args(); }; } diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp index 1891d551..845f0410 100644 --- a/src/xinterpreter.cpp +++ b/src/xinterpreter.cpp @@ -17,10 +17,8 @@ #include "xinput.hpp" #include "xinspect.hpp" -#ifndef EMSCRIPTEN #include "xmagics/os.hpp" #include "xmagics/xassist.hpp" -#endif #include "xparser.hpp" #include "xsystem.hpp" @@ -28,7 +26,9 @@ using Args = std::vector; void* createInterpreter(const Args &ExtraArgs = {}) { Args ClangArgs = {/*"-xc++"*/"-v"}; // ? {"-Xclang", "-emit-llvm-only", "-Xclang", "-diagnostic-log-file", "-Xclang", "-", "-xc++"}; -#ifndef EMSCRIPTEN +#ifdef EMSCRIPTEN + ClangArgs.push_back("-std=c++20"); +#else if (std::find_if(ExtraArgs.begin(), ExtraArgs.end(), [](const std::string& s) { return s == "-resource-dir";}) == ExtraArgs.end()) { std::string resource_dir = Cpp::DetectResourceDir(); @@ -99,8 +99,7 @@ __get_cxx_version () auto cxx_version = Cpp::Evaluate(code); return std::to_string(cxx_version); #else - constexpr int cxx_version = 20; - return std::to_string(cxx_version); + return "20"; #endif } @@ -115,9 +114,11 @@ __get_cxx_version () createInterpreter(Args(argv ? argv + 1 : argv, argv + argc)); m_version = get_stdopt(); redirect_output(); +#ifndef EMSCRIPTEN init_includes(); init_preamble(); init_magic(); +#endif } interpreter::~interpreter() @@ -362,9 +363,7 @@ __get_cxx_version () void interpreter::init_includes() { -#ifndef EMSCRIPTEN Cpp::AddIncludePath((xeus::prefix_path() + "/include/").c_str()); -#endif } void interpreter::init_preamble() @@ -383,9 +382,7 @@ __get_cxx_version () // preamble_manager["magics"].get_cast().register_magic("timeit", // timeit(&m_interpreter)); // preamble_manager["magics"].get_cast().register_magic("python", pythonexec()); -#ifndef EMSCRIPTEN preamble_manager["magics"].get_cast().register_magic("xassist", xassist()); preamble_manager["magics"].get_cast().register_magic("file", writefile()); -#endif } } diff --git a/src/xinterpreter_wasm.cpp b/src/xinterpreter_wasm.cpp index c3980773..7746c7b6 100644 --- a/src/xinterpreter_wasm.cpp +++ b/src/xinterpreter_wasm.cpp @@ -17,13 +17,7 @@ namespace xcpp { wasm_interpreter::wasm_interpreter() - : interpreter(create_args().size(), create_args().data()) + : interpreter(0, nullptr) { } - - std::vector wasm_interpreter::create_args() - { - static std::vector Args = {"-Xclang", "-std=c++20"}; - return Args; - } } From b04f64123cb186a1968baf84dea17b2369222660 Mon Sep 17 00:00:00 2001 From: anutosh491 Date: Mon, 23 Dec 2024 04:37:40 +0530 Subject: [PATCH 5/6] revert changes --- src/xinterpreter.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp index 845f0410..012a454d 100644 --- a/src/xinterpreter.cpp +++ b/src/xinterpreter.cpp @@ -17,8 +17,10 @@ #include "xinput.hpp" #include "xinspect.hpp" +#ifndef EMSCRIPTEN #include "xmagics/os.hpp" #include "xmagics/xassist.hpp" +#endif #include "xparser.hpp" #include "xsystem.hpp" @@ -114,11 +116,9 @@ __get_cxx_version () createInterpreter(Args(argv ? argv + 1 : argv, argv + argc)); m_version = get_stdopt(); redirect_output(); -#ifndef EMSCRIPTEN init_includes(); init_preamble(); init_magic(); -#endif } interpreter::~interpreter() @@ -363,7 +363,9 @@ __get_cxx_version () void interpreter::init_includes() { +#ifndef EMSCRIPTEN Cpp::AddIncludePath((xeus::prefix_path() + "/include/").c_str()); +#endif } void interpreter::init_preamble() @@ -382,7 +384,9 @@ __get_cxx_version () // preamble_manager["magics"].get_cast().register_magic("timeit", // timeit(&m_interpreter)); // preamble_manager["magics"].get_cast().register_magic("python", pythonexec()); +#ifndef EMSCRIPTEN preamble_manager["magics"].get_cast().register_magic("xassist", xassist()); preamble_manager["magics"].get_cast().register_magic("file", writefile()); +#endif } } From 29e82ee3c13716afa1a64201590d603236179ce3 Mon Sep 17 00:00:00 2001 From: anutosh491 Date: Tue, 24 Dec 2024 10:27:26 +0530 Subject: [PATCH 6/6] remove init includes --- src/xinterpreter.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/src/xinterpreter.cpp b/src/xinterpreter.cpp index 012a454d..cc27be25 100644 --- a/src/xinterpreter.cpp +++ b/src/xinterpreter.cpp @@ -116,7 +116,6 @@ __get_cxx_version () createInterpreter(Args(argv ? argv + 1 : argv, argv + argc)); m_version = get_stdopt(); redirect_output(); - init_includes(); init_preamble(); init_magic(); } @@ -361,13 +360,6 @@ __get_cxx_version () publish_stream("stderr", s); } - void interpreter::init_includes() - { -#ifndef EMSCRIPTEN - Cpp::AddIncludePath((xeus::prefix_path() + "/include/").c_str()); -#endif - } - void interpreter::init_preamble() { //NOLINTBEGIN(cppcoreguidelines-owning-memory)