From 3463fe01f475ac6c2a5b6a35e6d6649b5474ee49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Fri, 24 Aug 2018 22:22:36 +0200 Subject: [PATCH 01/10] Test and document embedding with dynamically loaded libjulia. --- doc/src/manual/embedding.md | 172 +++++++++++++++++++++ test/embedding/Makefile | 16 +- test/embedding/embedding-test.jl | 30 +++- test/embedding/embeddingdl-cfunctions.jl | 1 + test/embedding/embeddingdl.c | 185 +++++++++++++++++++++++ 5 files changed, 397 insertions(+), 7 deletions(-) create mode 100644 test/embedding/embeddingdl-cfunctions.jl create mode 100644 test/embedding/embeddingdl.c diff --git a/doc/src/manual/embedding.md b/doc/src/manual/embedding.md index 90850b7136f03..fb26cdfb7baf3 100644 --- a/doc/src/manual/embedding.md +++ b/doc/src/manual/embedding.md @@ -459,3 +459,175 @@ jl_errorf("argument x = %d is too large", x); ``` where in this example `x` is assumed to be an integer. + +## Dynamic Loading of `libjulia` + +The method of embedding shown so far requires that the embedding +program is linked with `libjulia`. As a consequence `libjulia` and the +rest of the Julia runtime must be shipped with the embedding program, +alternatively the user of the program is required to have a matching +Julia installed, in order to use the program at all. + +In some situations, e.g. if you have a large application where Julia +is only an optional backend or plugin, this can be +problematic. Requiring users that don't need the Julia functionality +to have Julia (of the right version) installed is rather user +unfriendly, and the Julia runtime is large enough that bundling it +will likely have noticeable effects on the application size. + +If this is the case, an alternative strategy is to dynamically load +`libjulia` with `dlopen` (`LoadLibrary` in Windows). This means that +`libjulia` is loaded at runtime when it is needed, and those who do +not need Julia functionality also do not need to have Julia installed, +or bundled. + +This approach does not come for free, however. It needs a certain +amount of overhead related to the loading, and only a subset of the +functions in `julia.h` can be used. Specifically anything involving C +macros is unavailable, which severely limits the possibilities to +manage memory and the use of everything that depends on correct memory +management. This is not quite as crippling as it may sound, however, +as will be shown in the following sections. The key is to perform all +communication with the Julia code through `cfunction`s and pass data +using only types from C. + +### Overview + +To use a dynamically loaded `libjulia` involves a number of setup +steps. + +* Open `libjulia` with `dlopen`/`LoadLibrary`. +* Obtain function pointers to the functions needed from `libjulia`. +* Initialize Julia. +* Load the Julia code that is going to be used and create `cfunction` + pointers to the functions the embedding code will interact with. +* Retrieve the `cfunction` pointers from the embedding code. + +Each of these steps will be discussed in more detail but to see the +full picture, look at the `embeddingdl.c` program in the Julia source +tree in the `test/embedding/` folder. That code also includes error +handling, which is omitted from this presentation for brevity. + +### Load `libjulia` + +#### Linux + +Dynamic loading of `libjulia` is done with `dlopen`. +``` +#include +void *libjulia = dlopen("libjulia.so"); +``` +In order for the application to be able to find `libjulia.so`, it +needs to be in a directory listed in `LD_LIBRARY_PATH`. Alternatively +it can be in the `rpath` built into the executable or `dlopen` is +given an absolute path. In the latter case the embedding program can +determine the path from additional information, e.g. querying the +Julia binary (assuming it is in `PATH`) or relying on some environment +variable to point to the location of `libjulia`. + +#### Windows + +Dynamic loading of `libjulia` is done with `LoadLibrary`. +``` +#include +HMODULE libjulia = LoadLibrary("libjulia.dll"); +``` +In order for the application to be able to find `libjulia.dll`, it +must either be in the same directory as the executable, or in `Path`. + +### Retrieve Function Pointers from `libjulia` + +#### Linux + +Function pointers are retrieved from `libjulia` using `dlsym`. +``` +void (*p_jl_init)(void) = dlsym(libjulia, "jl_init"); +``` +The `jl_init` function can now be called by dereferencing the function +pointer, +``` +(*p_jl_init)(); +``` +Optionally the function pointer can be masqueraded to allow the same +syntax as the ordinary function call `jl_init()`. +``` +#define jl_init (*p_jl_init) +``` +This has to be done for each and every function from `libjulia` that +is needed, so it is advantageous to keep the list as short as possible. + +Some functions have arguments or return value of a Julia type, +e.g. `jl_value_t`. These types can be obtained by `#include "julia.h"` +but that would interfere with the masquerading trick and is not really +necessary. It is sufficient to declare them locally without the +definition of what is inside the `struct`s. +``` +typedef struct _jl_value_t jl_value_t; +``` +Not only functions can be found in `libjulia`, but also global +variables, e.g. the useful `jl_main_module`. These can also be +retrieved by `dlsym`. +``` +jl_module_t **p_jl_main_module = dlsym(libjulia, "jl_main_module"); +#define jl_main_module (*p_jl_main_module) +``` + +#### Windows + +The only difference from Linux is that `dlsym` is replaced by +`GetProcAddress`. + +#### Renaming of `jl_init` + +If Julia has been compiled with threading enabled, `jl_init` goes +under the name of `jl_init__threading`. The easiest way to find out is +to ask `dlsym` to resolve the symbols. +``` +void (*p_jl_init)(void) = dlsym(libjulia, "jl_init"); +if (!p_jl_init) + (*p_jl_init)(void) = dlsym(libjulia, "jl_init__threading"); +#define jl_init (*p_jl_init) +``` +This is also the case for `jl_init_with_image`. + +### Initialize Julia + +Note: In this and following sections it is assumed that function +pointers have been masqueraded as normal function calls. + +Initialization of Julia is the same as without dynamic loading of +`libjulia`. Normally it is sufficient to just do +``` +jl_init(); +``` +With dynamic loading it may be convenient to set up the Julia code +with the Julia function `include`. It is worth noting that this is not +configured like in the REPL (for embedding in general, not only with +dynamic loading) and if you want to use it the same way as in the REPL +to include code in the `Main` module, you need to define a method +yourself. +``` +jl_eval_string("include(x) = Base.include(Main, x)"); +``` + +### Creating and Retrieving `cfunction` Pointers + +The interaction between the embedding code and the Julia code is most +easily done by calling specific methods of Julia functions as +`cfunction`s and using C types for the data passed back and forth. +``` +jl_eval_string("const julia_sqrt = @cfunction(sqrt, Cdouble, (Cdouble,))"); +double (*p_julia_sqrt)(double) = jl_unbox_voidpointer( + jl_get_global(jl_main_module, jl_symbol("julia_sqrt"))); +printf("%e\n", (*p_julia_sqrt)(2.0)); +``` +The creation of the needed `cfunction` pointers, as well as loading of +the Julia code, is most convenient to do all at once in a Julia file +and including it in `Main`. +``` +jl_eval_string("include(\"setup.jl\")"); +``` +Retrieval of the `cfunction` pointers must be done one by one but the +lengthy invocation can of course be packaged into a helper +function. Optionally these function pointers can also be masqueraded +as regular functions. diff --git a/test/embedding/Makefile b/test/embedding/Makefile index df31c3735c9de..a3d5cf4d2cb3e 100644 --- a/test/embedding/Makefile +++ b/test/embedding/Makefile @@ -24,13 +24,14 @@ JULIA_CONFIG := $(JULIA) -e 'include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "jul CPPFLAGS_ADD := CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags) LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) +LDFLAGS_DL = -ldl DEBUGFLAGS += -g #============================================================================= -release: $(BIN)/embedding$(EXE) -debug: $(BIN)/embedding-debug$(EXE) +release: $(BIN)/embedding$(EXE) $(BIN)/embeddingdl$(EXE) +debug: $(BIN)/embedding-debug$(EXE) $(BIN)/embeddingdl-debug$(EXE) $(BIN)/embedding$(EXE): $(SRCDIR)/embedding.c $(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) @@ -38,6 +39,12 @@ $(BIN)/embedding$(EXE): $(SRCDIR)/embedding.c $(BIN)/embedding-debug$(EXE): $(SRCDIR)/embedding.c $(CC) $^ -o $@ $(CPPFLAGS_ADD) $(CPPFLAGS) $(CFLAGS_ADD) $(CFLAGS) $(LDFLAGS_ADD) $(LDFLAGS) $(DEBUGFLAGS) +$(BIN)/embeddingdl$(EXE): $(SRCDIR)/embeddingdl.c + $(CC) $^ -o $@ $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(LDFLAGS_DL) + +$(BIN)/embeddingdl-debug$(EXE): $(SRCDIR)/embeddingdl.c + $(CC) $^ -o $@ $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) $(LDFLAGS_DL) $(DEBUGFLAGS) + ifneq ($(abspath $(BIN)),$(abspath $(SRCDIR))) # for demonstration purposes, our demo code is also installed # in $BIN, although this would likely not be typical @@ -45,12 +52,13 @@ $(BIN)/LocalModule.jl: $(SRCDIR)/LocalModule.jl cp $< $@ endif -check: $(BIN)/embedding$(EXE) $(BIN)/LocalModule.jl - $(JULIA) --depwarn=error $(SRCDIR)/embedding-test.jl $< +check: $(BIN)/embedding$(EXE) $(BIN)/embeddingdl$(EXE) $(BIN)/LocalModule.jl + $(JULIA) --depwarn=error $(SRCDIR)/embedding-test.jl $^ @echo SUCCESS clean: -rm -f $(BIN)/embedding-debug$(EXE) $(BIN)/embedding$(EXE) + -rm -f $(BIN)/embeddingdl-debug$(EXE) $(BIN)/embeddingdl$(EXE) .PHONY: release debug clean check diff --git a/test/embedding/embedding-test.jl b/test/embedding/embedding-test.jl index 2be2842ac8967..75999be48c0e5 100644 --- a/test/embedding/embedding-test.jl +++ b/test/embedding/embedding-test.jl @@ -1,18 +1,19 @@ # This file is a part of Julia. License is MIT: https://julialang.org/license -# tests the output of the embedding example is correct +# tests that the output of the embedding examples are correct using Test +using Libdl if Sys.iswindows() # libjulia needs to be in the same directory as the embedding executable or in path ENV["PATH"] = string(Sys.BINDIR, ";", ENV["PATH"]) end -@test length(ARGS) == 1 +@test length(ARGS) == 3 @testset "embedding example" begin out = Pipe() err = Pipe() - p = run(pipeline(Cmd(ARGS), stdin=devnull, stdout=out, stderr=err), wait=false) + p = run(pipeline(Cmd(ARGS[1:1]), stdin=devnull, stdout=out, stderr=err), wait=false) close(out.in) close(err.in) out_task = @async readlines(out) @@ -26,3 +27,26 @@ end @test lines[9] == "calling new bar" @test lines[10] == " From worker 2:\tTaking over the world..." end + +if !Sys.iswindows() + # libjulia needs to be in LD_LIBRARY_PATH in order to dlopen it. + # On Windows it needs to be in the same directory as the embedding + # executable or in PATH, but that was arranged earlier. + libdir = dirname(abspath(Libdl.dlpath("libjulia"))) + ENV["LD_LIBRARY_PATH"] = string(libdir, ":", ENV["LD_LIBRARY_PATH"]) +end + +@testset "embedding dl" begin + out = Pipe() + err = Pipe() + p = run(pipeline(Cmd(ARGS[2:2]), stdin=devnull, stdout=out, stderr=err), wait=false) + close(out.in) + close(err.in) + out_task = @async readlines(out) + err = read(err, String) + @test err == "Intentional error: MethodError: no method matching this_function_has_no_methods()\n" + @test success(p) + lines = fetch(out_task) + @test length(lines) == 1 + @test lines[1] == "1.414214e+00" +end diff --git a/test/embedding/embeddingdl-cfunctions.jl b/test/embedding/embeddingdl-cfunctions.jl new file mode 100644 index 0000000000000..ab88917bce1e9 --- /dev/null +++ b/test/embedding/embeddingdl-cfunctions.jl @@ -0,0 +1 @@ +const julia_sqrt = @cfunction(sqrt, Cdouble, (Cdouble,)) diff --git a/test/embedding/embeddingdl.c b/test/embedding/embeddingdl.c new file mode 100644 index 0000000000000..a09cfc5585fd4 --- /dev/null +++ b/test/embedding/embeddingdl.c @@ -0,0 +1,185 @@ +#include +#include +#include + +#ifdef _OS_WINDOWS_ +#include +#else +#include +#endif + +// Load a minimal subset of the functions from julia.h dynamically and +// masquerade the function pointers to look like the original +// functions. + +// We could get the definitions of these structs from +// `#include "julia.h"`, but we only need to declare them and +// including julia.h would interfere with the masquerading trick. +typedef struct _jl_value_t jl_value_t; +typedef struct _jl_sym_t jl_sym_t; +typedef struct _jl_module_t jl_module_t; + +// Declare pointers to the functions we need to load from libjulia. +// Obviously these signatures must match the corresponding functions +// in the julia.h that libjulia was built from. +static void (*p_jl_init)(void); +static int (*p_jl_is_initialized)(void); +static void (*p_jl_atexit_hook)(int); +static jl_value_t *(*p_jl_eval_string)(const char *); +static jl_sym_t *(*p_jl_symbol)(const char *); +static jl_module_t **p_jl_main_module; +static jl_value_t *(*p_jl_get_global)(jl_module_t *, jl_sym_t *); +static void *(*p_jl_unbox_voidpointer)(jl_value_t *); +static jl_value_t *(*p_jl_exception_occurred)(void); + +// Helper function to extract function pointers from the dynamically +// loaded libjulia. +#if _OS_WINDOWS_ +static void *load_function(HMODULE libjulia, const char *name, int *success) +#else +static void *load_function(void *libjulia, const char *name, int *success) +#endif +{ +#if _OS_WINDOWS_ + void *p = GetProcAddress(libjulia, name); +#else + void *p = dlsym(libjulia, name); +#endif + + // Unfortunately Julia renames jl_init to jl_init__threading if + // Julia is compiled with threading support, so we have to check + // which of these is available, or otherwise query Julia in some + // other way (issue #28824). + if (!p && strcmp(name, "jl_init") == 0) { +#if _OS_WINDOWS_ + p = GetProcAddress(libjulia, "jl_init__threading"); +#else + p = dlsym(libjulia, "jl_init__threading"); +#endif + } + + if (!p) { + fprintf(stderr, "%s not found in libjulia.\n", name); + *success = 0; + } + + return p; +} + +// Open libjulia and extract pointers to the needed functions. +static int load_julia_functions() +{ +#if _OS_WINDOWS_ + // libjulia.dll needs to be in the same directory as the + // executable or in PATH. + const char *library_name = "libjulia.dll"; + HMODULE libjulia = LoadLibrary(library_name); +#else + // libjulia.so needs to be in LD_LIBRARY_PATH. It could also be in + // rpath (but that kind of defeats the purpose of dynamically + // loading libjulia) or an absolute path could be given, computed + // from other information. + const char *library_name = "libjulia.so"; + void *libjulia = dlopen(library_name, RTLD_LAZY | RTLD_GLOBAL); +#endif + + if (!libjulia) { + fprintf(stderr, "Failed to load libjulia.\n"); + return 0; + } + + int success = 1; + p_jl_init = load_function(libjulia, "jl_init", &success); + p_jl_is_initialized = load_function(libjulia, "jl_is_initialized", &success); + p_jl_atexit_hook = load_function(libjulia, "jl_atexit_hook", &success); + p_jl_eval_string = load_function(libjulia, "jl_eval_string", &success); + p_jl_symbol = load_function(libjulia, "jl_symbol", &success); + p_jl_main_module = load_function(libjulia, "jl_main_module", &success); + p_jl_get_global = load_function(libjulia, "jl_get_global", &success); + p_jl_unbox_voidpointer = load_function(libjulia, "jl_unbox_voidpointer", &success); + p_jl_exception_occurred = load_function(libjulia, "jl_exception_occurred", &success); + + return success; +} + +// Masquerade the dynamically loaded function pointers as regular functions. +#define jl_init (*p_jl_init) +#define jl_is_initialized (*p_jl_is_initialized) +#define jl_atexit_hook (*p_jl_atexit_hook) +#define jl_eval_string (*p_jl_eval_string) +#define jl_symbol (*p_jl_symbol) +#define jl_main_module (*p_jl_main_module) +#define jl_get_global (*p_jl_get_global) +#define jl_unbox_voidpointer (*p_jl_unbox_voidpointer) +#define jl_exception_occurred (*p_jl_exception_occurred) + +// Helper function to retrieve pointers to cfunctions on the Julia side. +static void *get_cfunction_pointer(const char *name) +{ + void *p = 0; + jl_value_t *boxed_pointer = jl_get_global(jl_main_module, jl_symbol(name)); + + if (boxed_pointer != 0) { + p = jl_unbox_voidpointer(boxed_pointer); + } + + if (!p) { + fprintf(stderr, "cfunction pointer %s not available.\n", name); + } + + return p; +} + +// jl_eval_string with error checking. +static int checked_eval_string(const char *command, const char *error_message) +{ + jl_eval_string(command); + + if (jl_exception_occurred()) { + const char *p = jl_unbox_voidpointer(jl_eval_string("pointer(sprint(showerror, ccall(:jl_exception_occurred, Any, ())))")); + fprintf(stderr, "%s%s\n", error_message, p); + return 0; + } + + return 1; +} + +int main() +{ + if (!load_julia_functions()) + return EXIT_FAILURE; + + jl_init(); + if (!jl_is_initialized()) { + fprintf(stderr, "jl_init failed."); + jl_atexit_hook(EXIT_FAILURE); + return EXIT_FAILURE; + } + + // `include` is not available out of the box from the embedding + // environment, so we add it here (issue #28825). + checked_eval_string("include(x) = Base.include(Main, x)", + "Failed to define include: "); + + if (!checked_eval_string("include(\"embeddingdl-cfunctions.jl\")", + "Failed to load cfunctions: ")) { + jl_atexit_hook(EXIT_FAILURE); + return EXIT_FAILURE; + } + + double (*p_julia_sqrt)(double) = get_cfunction_pointer("julia_sqrt"); + if (!p_julia_sqrt) { + jl_atexit_hook(EXIT_FAILURE); + return EXIT_FAILURE; + } + + printf("%e\n", (*p_julia_sqrt)(2.0)); + fflush(stdout); + + // Handling exceptions gracefully + checked_eval_string("function this_function_has_no_methods end; this_function_has_no_methods()", "Intentional error: "); + + int ret = EXIT_SUCCESS; + jl_atexit_hook(ret); + return ret; +} From c83a0d4e8757a138e9de7999be6634ff5f6aee10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Sat, 25 Aug 2018 17:18:49 +0200 Subject: [PATCH 02/10] Set up LD_LIBRARY_PATH better. --- test/embedding/embedding-test.jl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/embedding/embedding-test.jl b/test/embedding/embedding-test.jl index 75999be48c0e5..c718e0b940ce8 100644 --- a/test/embedding/embedding-test.jl +++ b/test/embedding/embedding-test.jl @@ -33,7 +33,11 @@ if !Sys.iswindows() # On Windows it needs to be in the same directory as the embedding # executable or in PATH, but that was arranged earlier. libdir = dirname(abspath(Libdl.dlpath("libjulia"))) - ENV["LD_LIBRARY_PATH"] = string(libdir, ":", ENV["LD_LIBRARY_PATH"]) + if haskey(ENV, "LD_LIBRARY_PATH") + ENV["LD_LIBRARY_PATH"] = string(libdir, ":", ENV["LD_LIBRARY_PATH"]) + else + ENV["LD_LIBRARY_PATH"] = libdir + end end @testset "embedding dl" begin From b09b7faa69cf7a86d5b96ca416a30e2b12eb8574 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Sun, 26 Aug 2018 00:19:27 +0200 Subject: [PATCH 03/10] Improve OS conditionals. --- test/embedding/Makefile | 9 ++++++++- test/embedding/embeddingdl.c | 12 ++++++++---- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/test/embedding/Makefile b/test/embedding/Makefile index a3d5cf4d2cb3e..0574a3270a6a6 100644 --- a/test/embedding/Makefile +++ b/test/embedding/Makefile @@ -24,7 +24,14 @@ JULIA_CONFIG := $(JULIA) -e 'include(joinpath(Sys.BINDIR, Base.DATAROOTDIR, "jul CPPFLAGS_ADD := CFLAGS_ADD = $(shell $(JULIA_CONFIG) --cflags) LDFLAGS_ADD = -lm $(shell $(JULIA_CONFIG) --ldflags --ldlibs) -LDFLAGS_DL = -ldl + +BUILD_OS := $(shell uname) +ifeq ($(BUILD_OS), Linux) +LDFLAGS_DL := -ldl +endif +ifeq ($(BUILD_OS), Darwin) +LDFLAGS_DL := -ldl +endif DEBUGFLAGS += -g diff --git a/test/embedding/embeddingdl.c b/test/embedding/embeddingdl.c index a09cfc5585fd4..ae20c7fe7a04c 100644 --- a/test/embedding/embeddingdl.c +++ b/test/embedding/embeddingdl.c @@ -34,13 +34,13 @@ static jl_value_t *(*p_jl_exception_occurred)(void); // Helper function to extract function pointers from the dynamically // loaded libjulia. -#if _OS_WINDOWS_ +#ifdef _OS_WINDOWS_ static void *load_function(HMODULE libjulia, const char *name, int *success) #else static void *load_function(void *libjulia, const char *name, int *success) #endif { -#if _OS_WINDOWS_ +#ifdef _OS_WINDOWS_ void *p = GetProcAddress(libjulia, name); #else void *p = dlsym(libjulia, name); @@ -51,7 +51,7 @@ static void *load_function(void *libjulia, const char *name, int *success) // which of these is available, or otherwise query Julia in some // other way (issue #28824). if (!p && strcmp(name, "jl_init") == 0) { -#if _OS_WINDOWS_ +#ifdef _OS_WINDOWS_ p = GetProcAddress(libjulia, "jl_init__threading"); #else p = dlsym(libjulia, "jl_init__threading"); @@ -69,7 +69,7 @@ static void *load_function(void *libjulia, const char *name, int *success) // Open libjulia and extract pointers to the needed functions. static int load_julia_functions() { -#if _OS_WINDOWS_ +#ifdef _OS_WINDOWS_ // libjulia.dll needs to be in the same directory as the // executable or in PATH. const char *library_name = "libjulia.dll"; @@ -79,7 +79,11 @@ static int load_julia_functions() // rpath (but that kind of defeats the purpose of dynamically // loading libjulia) or an absolute path could be given, computed // from other information. +#ifdef __APPLE__ + const char *library_name = "libjulia.dylib"; +#else const char *library_name = "libjulia.so"; +#endif void *libjulia = dlopen(library_name, RTLD_LAZY | RTLD_GLOBAL); #endif From 7d6bcd10b7eb971371eb00f890ce95a8609b5f19 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Sun, 26 Aug 2018 11:57:43 +0200 Subject: [PATCH 04/10] Define _OS_WINDOWS_ --- test/embedding/embedding.c | 5 +++++ test/embedding/embeddingdl.c | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/test/embedding/embedding.c b/test/embedding/embedding.c index 52498b2648135..554b3414efcdc 100644 --- a/test/embedding/embedding.c +++ b/test/embedding/embedding.c @@ -6,6 +6,11 @@ JULIA_DEFINE_FAST_TLS() // only define this once, in an executable +// Extracted from src/support/platform.h +#if defined(_WIN32) || defined(_WIN64) +#define _OS_WINDOWS_ +#endif + #ifdef _OS_WINDOWS_ __declspec(dllexport) __cdecl #endif diff --git a/test/embedding/embeddingdl.c b/test/embedding/embeddingdl.c index ae20c7fe7a04c..b9e7876c064b9 100644 --- a/test/embedding/embeddingdl.c +++ b/test/embedding/embeddingdl.c @@ -2,6 +2,11 @@ #include #include +// Extracted from src/support/platform.h +#if defined(_WIN32) || defined(_WIN64) +#define _OS_WINDOWS_ +#endif + #ifdef _OS_WINDOWS_ #include #else From b048aa3bcfae7557b585f5362a6102bfdc36e3ac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Sun, 26 Aug 2018 14:40:27 +0200 Subject: [PATCH 05/10] Tweak the Makefile since Appveyor's make doesn't seem to support $^. --- doc/src/manual/embedding.md | 7 ++++++- test/embedding/Makefile | 6 ++++-- test/embedding/embedding-test.jl | 2 +- 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/doc/src/manual/embedding.md b/doc/src/manual/embedding.md index fb26cdfb7baf3..3489c3ed34bb5 100644 --- a/doc/src/manual/embedding.md +++ b/doc/src/manual/embedding.md @@ -525,6 +525,11 @@ determine the path from additional information, e.g. querying the Julia binary (assuming it is in `PATH`) or relying on some environment variable to point to the location of `libjulia`. +#### Mac + +Same as Linux with the exception that the library is called +`libjulia.dylib` instead of `libjulia.so`. + #### Windows Dynamic loading of `libjulia` is done with `LoadLibrary`. @@ -537,7 +542,7 @@ must either be in the same directory as the executable, or in `Path`. ### Retrieve Function Pointers from `libjulia` -#### Linux +#### Linux and Mac Function pointers are retrieved from `libjulia` using `dlsym`. ``` diff --git a/test/embedding/Makefile b/test/embedding/Makefile index 0574a3270a6a6..8f8eb7d344f4a 100644 --- a/test/embedding/Makefile +++ b/test/embedding/Makefile @@ -57,10 +57,12 @@ ifneq ($(abspath $(BIN)),$(abspath $(SRCDIR))) # in $BIN, although this would likely not be typical $(BIN)/LocalModule.jl: $(SRCDIR)/LocalModule.jl cp $< $@ +$(BIN)/embedding-cfunctions.jl: $(SRCDIR)/embedding-cfunctions.jl + cp $< $@ endif -check: $(BIN)/embedding$(EXE) $(BIN)/embeddingdl$(EXE) $(BIN)/LocalModule.jl - $(JULIA) --depwarn=error $(SRCDIR)/embedding-test.jl $^ +check: $(BIN)/embedding$(EXE) $(BIN)/embeddingdl$(EXE) $(BIN)/LocalModule.jl $(BIN)/embeddingdl-cfunctions.jl + $(JULIA) --depwarn=error $(SRCDIR)/embedding-test.jl $(BIN)/embedding$(EXE) $(BIN)/embeddingdl$(EXE) @echo SUCCESS clean: diff --git a/test/embedding/embedding-test.jl b/test/embedding/embedding-test.jl index c718e0b940ce8..d60081b956694 100644 --- a/test/embedding/embedding-test.jl +++ b/test/embedding/embedding-test.jl @@ -9,7 +9,7 @@ if Sys.iswindows() ENV["PATH"] = string(Sys.BINDIR, ";", ENV["PATH"]) end -@test length(ARGS) == 3 +@test length(ARGS) == 2 @testset "embedding example" begin out = Pipe() err = Pipe() From ce8620a7a3e8100c1303be86b4c776fed8b4658d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Sun, 26 Aug 2018 19:54:53 +0200 Subject: [PATCH 06/10] Fix typo and update appveyor.yml. --- appveyor.yml | 2 +- test/embedding/.gitignore | 2 ++ test/embedding/Makefile | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 6f0ca2d04c40a..ab2512ea471ad 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -64,4 +64,4 @@ test_script: - cd julia-* && .\bin\julia.exe --check-bounds=yes share\julia\test\runtests.jl all && .\bin\julia.exe --check-bounds=yes share\julia\test\runtests.jl LibGit2/online Pkg/pkg download - cd .. - - usr\bin\julia usr\share\julia\test\embedding\embedding-test.jl test\embedding\embedding.exe + - usr\bin\julia usr\share\julia\test\embedding\embedding-test.jl test\embedding\embedding.exe test\embedding\embeddingdl.exe diff --git a/test/embedding/.gitignore b/test/embedding/.gitignore index 4ce246ef16619..571dc421ff09e 100644 --- a/test/embedding/.gitignore +++ b/test/embedding/.gitignore @@ -1,2 +1,4 @@ /embedding /embedding-debug +/embeddingdl +/embeddingdl-debug diff --git a/test/embedding/Makefile b/test/embedding/Makefile index 8f8eb7d344f4a..e29c2f55d4fa7 100644 --- a/test/embedding/Makefile +++ b/test/embedding/Makefile @@ -57,7 +57,7 @@ ifneq ($(abspath $(BIN)),$(abspath $(SRCDIR))) # in $BIN, although this would likely not be typical $(BIN)/LocalModule.jl: $(SRCDIR)/LocalModule.jl cp $< $@ -$(BIN)/embedding-cfunctions.jl: $(SRCDIR)/embedding-cfunctions.jl +$(BIN)/embeddingdl-cfunctions.jl: $(SRCDIR)/embeddingdl-cfunctions.jl cp $< $@ endif From fee921e6673f2f03d093f49dbbf8e2f93a64c49d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Sun, 26 Aug 2018 22:54:29 +0200 Subject: [PATCH 07/10] Find the included file relative to the executable. --- test/embedding/embeddingdl.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/embedding/embeddingdl.c b/test/embedding/embeddingdl.c index b9e7876c064b9..3566b34b30319 100644 --- a/test/embedding/embeddingdl.c +++ b/test/embedding/embeddingdl.c @@ -170,7 +170,8 @@ int main() checked_eval_string("include(x) = Base.include(Main, x)", "Failed to define include: "); - if (!checked_eval_string("include(\"embeddingdl-cfunctions.jl\")", + // Find the included file relative to the `embeddingdl` executable. + if (!checked_eval_string("include(joinpath(dirname(unsafe_string(Base.JLOptions().julia_bin)), \"embeddingdl-cfunctions.jl\"))", "Failed to load cfunctions: ")) { jl_atexit_hook(EXIT_FAILURE); return EXIT_FAILURE; From bec120347f3b3bbb0bc40a1402f548ed44dd26e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Mon, 27 Aug 2018 07:33:44 +0200 Subject: [PATCH 08/10] Test output value more robustly. --- test/embedding/embedding-test.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/embedding/embedding-test.jl b/test/embedding/embedding-test.jl index d60081b956694..55057b56b78c0 100644 --- a/test/embedding/embedding-test.jl +++ b/test/embedding/embedding-test.jl @@ -52,5 +52,5 @@ end @test success(p) lines = fetch(out_task) @test length(lines) == 1 - @test lines[1] == "1.414214e+00" + @test parse(Float64, lines[1]) == 1.414214 end From d7b5d17f2861dd09d91c6f07f322ff5869fdb7e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Mon, 27 Aug 2018 19:52:26 +0200 Subject: [PATCH 09/10] Chomp for CRLF resistance. --- test/embedding/embedding-test.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/embedding/embedding-test.jl b/test/embedding/embedding-test.jl index 55057b56b78c0..e7bb809fae16e 100644 --- a/test/embedding/embedding-test.jl +++ b/test/embedding/embedding-test.jl @@ -48,7 +48,7 @@ end close(err.in) out_task = @async readlines(out) err = read(err, String) - @test err == "Intentional error: MethodError: no method matching this_function_has_no_methods()\n" + @test chomp(err) == "Intentional error: MethodError: no method matching this_function_has_no_methods()" @test success(p) lines = fetch(out_task) @test length(lines) == 1 From 1c175f1c6db774ca09bca691635dd3bd20c88f21 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gunnar=20Farneb=C3=83=C2=A4ck?= Date: Fri, 7 Dec 2018 22:21:43 +0100 Subject: [PATCH 10/10] Correct retrieval of string. --- test/embedding/embeddingdl.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/embedding/embeddingdl.c b/test/embedding/embeddingdl.c index 3566b34b30319..a302a33727a80 100644 --- a/test/embedding/embeddingdl.c +++ b/test/embedding/embeddingdl.c @@ -35,6 +35,7 @@ static jl_sym_t *(*p_jl_symbol)(const char *); static jl_module_t **p_jl_main_module; static jl_value_t *(*p_jl_get_global)(jl_module_t *, jl_sym_t *); static void *(*p_jl_unbox_voidpointer)(jl_value_t *); +static const char *(*p_jl_string_ptr)(jl_value_t *); static jl_value_t *(*p_jl_exception_occurred)(void); // Helper function to extract function pointers from the dynamically @@ -106,6 +107,7 @@ static int load_julia_functions() p_jl_main_module = load_function(libjulia, "jl_main_module", &success); p_jl_get_global = load_function(libjulia, "jl_get_global", &success); p_jl_unbox_voidpointer = load_function(libjulia, "jl_unbox_voidpointer", &success); + p_jl_string_ptr = load_function(libjulia, "jl_string_ptr", &success); p_jl_exception_occurred = load_function(libjulia, "jl_exception_occurred", &success); return success; @@ -120,6 +122,7 @@ static int load_julia_functions() #define jl_main_module (*p_jl_main_module) #define jl_get_global (*p_jl_get_global) #define jl_unbox_voidpointer (*p_jl_unbox_voidpointer) +#define jl_string_ptr (*p_jl_string_ptr) #define jl_exception_occurred (*p_jl_exception_occurred) // Helper function to retrieve pointers to cfunctions on the Julia side. @@ -145,7 +148,7 @@ static int checked_eval_string(const char *command, const char *error_message) jl_eval_string(command); if (jl_exception_occurred()) { - const char *p = jl_unbox_voidpointer(jl_eval_string("pointer(sprint(showerror, ccall(:jl_exception_occurred, Any, ())))")); + const char *p = jl_string_ptr(jl_eval_string("sprint(showerror, ccall(:jl_exception_occurred, Any, ()))")); fprintf(stderr, "%s%s\n", error_message, p); return 0; }