Avoid creating a new internal interpreter if DLite is called from Python

.github/workflows/ci_tests.yml

@@ -63,27 +63,39 @@ jobs:
         python --version
         pip freeze
 
-    - name: configure
+    - name: Configure
       run: |
         Python3_ROOT=$(python3 -c 'import sys; print(sys.exec_prefix)') \
         CFLAGS='-Wno-missing-field-initializers' \
         cmake -B build . -DFORCE_EXAMPLES=ON -DWITH_FORTRAN=YES
 
-    - name: make
+    - name: Build
       run: cmake --build .
       working-directory: build
 
-    - name: install
+    - name: Install
       run: cmake --install .
       working-directory: build
 
-    - name: make test
+    - name: Test
       env:
         DLITE_PYDEBUG: ""
       run: ctest || ctest --rerun-failed --output-on-failure -V
       working-directory: build
 
-    - name: build Linux wheel
+    - name: Test with all behavior changes disabled
+      env:
+        DLITE_BEHAVIOR: OFF
+      run: ctest || ctest --rerun-failed --output-on-failure -V
+      working-directory: build
+
+    - name: Test with all behavior changes enabled
+      env:
+        DLITE_BEHAVIOR: ON
+      run: ctest || ctest --rerun-failed --output-on-failure -V
+      working-directory: build
+
+    - name: Build Linux wheel
       run: python3 -m pip wheel -w dist ./python
 
     - name: Install python package and test the installation

doc/contributors_guide/code_changes.md

@@ -71,10 +71,13 @@ Whether to enable a behavior can be configured in several ways:
 
       export DLITE_BEHAVIOR_<NAME>=1
 
-  The empty string or any of the following values will enable the behavior:
+  You can use the environment variable `DLITE_BEHAVIOR` to initialise all
+  behavior variables.  This is e.g. useful for testing.
+
+  The empty string or any of the following values will enable the behavior change:
   "true", ".true.", "on", "yes", 1
 
-  Any of the following values will disable the behavior:
+  Any of the following values will disable the behavior change:
   "false", ".false.", "off", "no", 0
 
 A warning will automatically be issued if a behavior is not selected explicitly.

doc/user_guide/configure_behavior_changes.md

@@ -0,0 +1,24 @@
+Configure behavior changes
+==========================
+When new backward incompatible changes are implemented in DLite, the user will in
+most cases get a warning of the form
+
+> Warning: Behavior `<NAME>` is not configured. It will be enabled by default from v0.7.0. See https://sintef.github.io/dlite/user_guide/configure_behavior_changes.html for more info.
+
+where `<NAME>` will be the name of the behavior that is going to be changed.
+
+You can turn off this warning by configuring whether you want the old (deprecated) behavior or the new behavior by defining the environment variable `DLITE_BEHAVIOR_<NAME>`.
+
+If `DLITE_BEHAVIOR_<NAME>` is defined without a value or its value is one of "true", ".true.", "on", "yes" or 1, the behavior change will be enabled.
+
+If `DLITE_BEHAVIOR_<NAME>` is one of "false", ".false.", "off", "no" or 0, the behavior change will be disabled.
+
+It is also possible to enable/disable all defined behavior changes by setting the environment variable `DLITE_BEHAVIOR`. This can be overwritten for individual behavior changes by setting `DLITE_BEHAVIOR_<NAME>`.
+
+Currently the different scheduled behavior changes are only documented in the [behavior table] in the source code.  In this table you can find a description of the behavior change `<NAME>` in addition to the version it was added, the version it will be turned on by default and the version it is planned for removal.
+
+
+
+
+
+[behavior table]: https://github.com/SINTEF/dlite/blob/6f38fb49ea2dbb6252bf44cd6d239a6adff6050e/src/dlite-behavior.c#L23

doc/user_guide/environment_variables.md

@@ -52,14 +52,18 @@ DLite-specific environment variables
   - **DLITE_ATEXIT_FREE**: Free memory at exit.  This might be useful to avoid
     getting false positive when tracking down memory leaks with tools like valgrind.
 
-  - **DLITE_BEHAVIOR_<name>**: Enables/disables the behavior `<name>`.
+  - **DLITE_BEHAVIOR**: Enables/disables all behavior changes by default.
 
-    The empty string or any of the following values will enable the behavior:
+    The empty string or any of the following values will enable the behaviors:
     "true", ".true.", "on", "yes", 1
 
-    Any of the following values will disable the behavior:
+    Any of the following values will disable the behaviors:
     "false", ".false.", "off", "no", 0
 
+  - **DLITE_BEHAVIOR_<name>**: Enables/disables the behavior `<name>`.
+    This overrides `DLITE_BEHAVIOR` for the named behavior.  The value
+    has the same meaning as for `DLITE_BEHAVIOR`.
+
 
 ### Specific paths
 These environment variables can be used to provide additional search

doc/user_guide/index.rst

@@ -15,6 +15,7 @@ User Guide
    code_generation
    storage_plugins
    storage_plugins_mongodb
+   configure_behavior_changes
    environment_variables
    features
    vocabulary

src/dlite-behavior.c

@@ -22,8 +22,9 @@
 //  description
 static DLiteBehavior behavior_table[] = {
   { "singleInterpreter",           "0.5.17",  "0.7.0",   "0.9.0",
-    "Plugins are executed by the calling interpreter when DLite is called "
-    "from Python.", -1 },
+    "Evaluate Python plugins from calling interpreter when DLite is called "
+    "from Python.  The old behavior is to call the plugins from an internal "
+    "interpreter", -1 },
   { NULL,                          NULL,      NULL,      NULL,     NULL, 0 }
 };
 
@@ -49,13 +50,16 @@ void dlite_behavior_table_init(void)
   if (!initialised) {
     DLiteBehavior *b = (DLiteBehavior *)behavior_table;
     while (b->name) {
+      const char *env;
       b->value = -1;
 
-      /* Check environment */
+      /* Check environment variable: DLITE_BEHAVIOR */
+      if ((env = getenv("DLITE_BEHAVIOR"))) b->value = (*env) ? atob(env) : 1;
+
+      /* Check environment variable: DLITE_BEHAVIOR_<name> */
       char buf[64];
       snprintf(buf, sizeof(buf), "DLITE_BEHAVIOR_%s", b->name);
-      const char *env = getenv(buf);
-      if (env) b->value = (*env) ? atob(env) : 1;
+      if ((env = getenv(buf))) b->value = (*env) ? atob(env) : 1;
 
       /* Warn if behavior is expected to be removed. */
       if (strcmp_semver(dlite_get_version(), b->version_remove) >= 0)
@@ -115,7 +119,7 @@ const DLiteBehavior *dlite_behavior_record(const char *name)
 int dlite_behavior_get(const char *name)
 {
   DLiteBehavior *b = (DLiteBehavior *)dlite_behavior_record(name);
-  if (!b) return dlite_err(dliteKeyError, "No behavior with name: %s", name);
+  if (!b) return dlite_err(dliteNameError, "No behavior with name: %s", name);
 
   /* If value is unset, enable behavior if DLite version > version_new */
   if (b->value < 0) {
@@ -124,14 +128,8 @@ int dlite_behavior_get(const char *name)
 
     dlite_warn("Behavior `%s` is not configured. "
                "It will be enabled by default from v%s. "
-               "The old behavior is scheduled for removal in v%s.\n\n"
-               "To configure the behavior, set environment variable "
-               "`DLITE_BEHAVIOR_%s` to a boolean value.  True will "
-               "enable the new behavior and false will disable it.\n\n"
-               "You can also configure it from Python by setting "
-               "`dlite.Behavior.%s` or from C by calling "
-               "`dlite_behavior_set()`.",
-               b->name, b->version_new, b->version_remove, b->name, b->name);
+               "See https://sintef.github.io/dlite/user_guide/configure_behavior_changes.html for more info.",
+               b->name, b->version_new);
   }
 
   assert(b->value >= 0);
@@ -147,7 +145,7 @@ int dlite_behavior_get(const char *name)
 int dlite_behavior_set(const char *name, int value)
 {
   DLiteBehavior *b = (DLiteBehavior *)dlite_behavior_record(name);
-  if (!b) return dlite_err(dliteKeyError, "No behavior with name: %s", name);
+  if (!b) return dlite_err(dliteNameError, "No behavior with name: %s", name);
   b->value = value;
   return 0;
 }

src/pyembed/dlite-pyembed.c

@@ -3,6 +3,7 @@
 #include "utils/strutils.h"
 #include "dlite-macros.h"
 #include "dlite-misc.h"
+#include "dlite-behavior.h"
 #include "dlite-pyembed.h"
 #include "dlite-python-storage.h"
 #include "dlite-python-mapping.h"
@@ -17,9 +18,6 @@
 #endif
 
 
-static int python_initialized = 0;
-
-
 /* Struct correlating Python exceptions with DLite errors */
 typedef struct {
   PyObject *exc;        /* Python exception */
@@ -100,7 +98,6 @@ PyObject *dlite_pyembed_exception(DLiteErrCode code)
   return PyExc_Exception;
 }
 
-
 /* Help function returning a constant pointer to a NULL-terminated
    array of ErrorCorrelation records. */
 static const ErrorCorrelation *error_correlations(void)
@@ -127,15 +124,19 @@ static const ErrorCorrelation *error_correlations(void)
   return g->errcorr;
 }
 
-/* Initialises the embedded Python environment. */
-void dlite_pyembed_initialise(void)
-{
-  if (!python_initialized) {
-    PyObject *sys=NULL, *sys_path=NULL, *path=NULL;
+/* Initialises the embedded Python environment.
 
-    Py_Initialize();
-    python_initialized = 1;
+   From DLite v0.6.0, this function will only initialise an new
+   internal Python interpreter if there are no initialised
+   interpreters in the process.  This means that if DLite is called
+   from Python, the plugins will be called from the calling Python
+   interpreter.
 
+   This function can be called more than once.
+ */
+void dlite_pyembed_initialise(void)
+{
+  if (!Py_IsInitialized() || !dlite_behavior_get("singleInterpreter")) {
     /*
       Python 3.8 and later implements the new Python
       Initialisation Configuration.
@@ -148,6 +149,7 @@ void dlite_pyembed_initialise(void)
       In DLite, we switch to the new Python Initialisation
       Configuration from Python 3.11.
     */
+    PyObject *sys=NULL, *sys_path=NULL, *path=NULL;
 #if PY_VERSION_HEX >= 0x030b0000  /* Python >= 3.11 */
     /* New Python Initialisation Configuration */
     PyStatus status;
@@ -160,15 +162,16 @@ void dlite_pyembed_initialise(void)
     config.user_site_directory = 1;
 
     /* If dlite is called from a python, reparse arguments to avoid
-       that they are stripped off...
-       Aren't we initialising a new interpreter? */
-    int argc=0;
-    wchar_t **argv=NULL;
-    Py_GetArgcArgv(&argc, &argv);
-    config.parse_argv = 1;
-    status = PyConfig_SetArgv(&config, argc, argv);
-    if (PyStatus_Exception(status))
-      FAIL("failed configuring pyembed arguments");
+       that they are stripped off... */
+    if (Py_IsInitialized()) {
+      int argc=0;
+      wchar_t **argv=NULL;
+      Py_GetArgcArgv(&argc, &argv);
+      config.parse_argv = 1;
+      status = PyConfig_SetArgv(&config, argc, argv);
+      if (PyStatus_Exception(status))
+        FAIL("failed configuring pyembed arguments");
+    }
 
     status = PyConfig_SetBytesString(&config, &config.program_name, "dlite");
     if (PyStatus_Exception(status))
@@ -182,6 +185,8 @@ void dlite_pyembed_initialise(void)
     /* Old Initialisation */
     wchar_t *progname;
 
+    Py_Initialize();
+
     if (!(progname = Py_DecodeLocale("dlite", NULL))) {
       dlite_err(1, "allocation/decoding failure");
       return;
@@ -213,9 +218,8 @@ void dlite_pyembed_initialise(void)
 int dlite_pyembed_finalise(void)
 {
   int status=0;
-  if (python_initialized) {
+  if (Py_IsInitialized()) {
     status = Py_FinalizeEx();
-    python_initialized = 0;
   } else {
     return dlite_errx(1, "cannot finalize Python before it is initialized");
   }

src/tests/python/test_pyembed.c

@@ -12,6 +12,12 @@
 typedef DLiteInstance *(*fun_t)(const char *id);
 
 
+MU_TEST(test_pyembed_initialise)
+{
+  dlite_pyembed_initialise();
+}
+
+
 MU_TEST(test_add_dll_path)
 {
   dlite_add_dll_path();
@@ -99,6 +105,7 @@ MU_TEST(test_finalize)
 
 MU_TEST_SUITE(test_suite)
 {
+  MU_RUN_TEST(test_pyembed_initialise);
   MU_RUN_TEST(test_add_dll_path);
   MU_RUN_TEST(test_load_modules);
   MU_RUN_TEST(test_get_address);