Merge pull request #1922 from garlick/jobspec_val

job-ingest: validate jobspec using python/jsonschema

README.md

@@ -45,7 +45,6 @@ jansson-devel	| libjansson-dev	| >= 2.6		|
 lz4-devel	| liblz4-dev		|			|
 hwloc-devel	| libhwloc-dev		| >= v1.11.1, < 2.0	|
 sqlite-devel	| libsqlite3-dev	| >= 3.0.0		|
-yaml-cpp-devel	| libyaml-cpp-dev	| >= 0.5.1		|
 lua		| lua5.1		| >= 5.1, < 5.3		|
 lua-devel	| liblua5.1-dev		| >= 5.1, < 5.3		|
 lua-posix 	| lua-posix             | 			| *1*

configure.ac

@@ -104,19 +104,6 @@ AS_CASE($ax_cv_c_compiler_vendor,
   ]
 )
 
-AC_PROG_CXX
-# Check compiler vendor for c++, need to temporarily update AC_LANG
-AC_LANG_PUSH([C++])
-AX_COMPILER_VENDOR
-AC_LANG_POP
-AS_CASE($ax_cv_cxx_compiler_vendor,
-  [clang | gnu], [
-    WARNING_CXXFLAGS=$WARNING_CFLAGS
-    AC_SUBST([WARNING_CXXFLAGS])
-  ]
-)
-AX_CXX_COMPILE_STDCXX([11], [noext], [mandatory])
-
 X_AC_ENABLE_SANITIZER
 AC_COMPILE_IFELSE([AC_LANG_PROGRAM(,[
   #include <dlfcn.h>
@@ -288,7 +275,6 @@ AX_LUA_HEADERS
 AX_LUA_LIBS
 X_AC_ZEROMQ
 X_AC_JANSSON
-X_AC_YAMLCPP
 PKG_CHECK_MODULES([HWLOC], [hwloc >= 1.11.1], [], [])
 PKG_CHECK_MODULES([LZ4], [liblz4], [], [])
 PKG_CHECK_MODULES([SQLITE], [sqlite3], [], [])
@@ -415,7 +401,6 @@ AC_CONFIG_FILES( \
   src/common/libcompat/Makefile \
   src/common/liboptparse/Makefile \
   src/common/libidset/Makefile \
-  src/common/libjobspec/Makefile \
   src/common/libtomlc99/Makefile \
   src/bindings/Makefile \
   src/bindings/lua/Makefile \

src/cmd/Makefile.am

@@ -11,10 +11,6 @@ AM_CPPFLAGS = \
 	-I$(top_builddir)/src/common/libflux \
 	$(ZMQ_CFLAGS) $(FLUX_SECURITY_CFLAGS) $(LIBSODIUM_CFLAGS)
 
-AM_CXXFLAGS = \
-	$(WARNING_CXXFLAGS) \
-	$(CODE_COVERAGE_CXXFLAGS)
-
 fluxcmd_ldadd = \
 	$(top_builddir)/src/common/libkvs/libkvs.la \
 	$(top_builddir)/src/common/libflux-internal.la \

src/cmd/flux.c

@@ -199,6 +199,9 @@ int main (int argc, char *argv[])
     environment_from_env (env, "FLUX_WREXECD_PATH",
                           flux_conf_get ("wrexecd_path", flags), 0);
 
+    if ((flags & CONF_FLAG_INTREE))
+        environment_push (env, "FLUX_CONF_INTREE", "1");
+
     environment_apply(env);
     optparse_set_data (p, "env", env);
 

src/common/Makefile.am

@@ -15,10 +15,6 @@ SUBDIRS = libtap \
 	  libkz \
           libsubprocess
 
-if ENABLE_JOBSPEC
-SUBDIRS += libjobspec
-endif
-
 AM_CFLAGS = $(WARNING_CFLAGS) $(CODE_COVERAGE_CFLAGS)
 AM_LDFLAGS = $(CODE_COVERAGE_LIBS)
 

src/common/libflux/Makefile.am

@@ -29,7 +29,9 @@ installed_conf_cppflags = \
 	-DINSTALLED_CMDHELP_PATTERN=\"${datadir}/flux/help.d/*.json\" \
 	-DINSTALLED_NO_DOCS_PATH=\"${datadir}/flux/.nodocs\" \
 	-DINSTALLED_RUNDIR=\"${runstatedir}/flux\" \
-	-DINSTALLED_BINDIR=\"$(fluxcmddir)\"
+	-DINSTALLED_BINDIR=\"$(fluxcmddir)\" \
+	-DINSTALLED_JOBSPEC_VALIDATE_PATH=\"${fluxlibexecdir}/validate-schema.py\" \
+	-DINSTALLED_JOBSPEC_SCHEMA_PATH=\"${datadir}/flux/schema/jobspec/jobspec.jsonschema\"
 
 intree_conf_cppflags = \
 	-DINTREE_MODULE_PATH=\"$(abs_top_builddir)/src/modules\" \
@@ -48,7 +50,9 @@ intree_conf_cppflags = \
 	-DINTREE_CMDHELP_PATTERN=\"${abs_top_builddir}/etc/flux/help.d/*.json\" \
 	-DINTREE_KEYDIR=\"${abs_top_builddir}/etc/flux\" \
 	-DINTREE_NO_DOCS_PATH=\"${abs_top_builddir}/etc/flux/.nodocs\" \
-	-DINTREE_BINDIR=\"${abs_top_builddir}/src/cmd\"
+	-DINTREE_BINDIR=\"${abs_top_builddir}/src/cmd\" \
+	-DINTREE_JOBSPEC_VALIDATE_PATH=\"${abs_top_srcdir}/src/modules/job-ingest/validators/validate-schema.py\" \
+	-DINTREE_JOBSPEC_SCHEMA_PATH=\"${abs_top_srcdir}/src/modules/job-ingest/schemas/jobspec.jsonschema\"
 
 
 fluxcoreinclude_HEADERS = \

src/common/libflux/conf.c

@@ -42,6 +42,10 @@ static struct config default_config[] = {
     { "no_docs_path",   INSTALLED_NO_DOCS_PATH,     INTREE_NO_DOCS_PATH },
     { "rundir",         INSTALLED_RUNDIR,           NULL },
     { "bindir",         INSTALLED_BINDIR,           INTREE_BINDIR },
+    { "jobspec_validate_path", INSTALLED_JOBSPEC_VALIDATE_PATH,
+                                            INTREE_JOBSPEC_VALIDATE_PATH },
+    { "jobspec_schema_path", INSTALLED_JOBSPEC_SCHEMA_PATH,
+                                            INTREE_JOBSPEC_SCHEMA_PATH },
     { NULL, NULL, NULL },
 };
 

src/common/libflux/future.c

@@ -61,6 +61,7 @@ struct flux_future {
     struct now_context *now;
     struct then_context *then;
     zlist_t *queue;
+    int refcount;
 };
 
 static void check_cb (flux_reactor_t *r, flux_watcher_t *w,
@@ -291,17 +292,17 @@ static struct future_result *future_result_errnum_create (int errnum,
  */
 void flux_future_destroy (flux_future_t *f)
 {
-    int saved_errno = errno;
-    if (f) {
+    if (f && (--f->refcount == 0)) {
+        int saved_errno = errno;
         aux_destroy (&f->aux);
         clear_result (&f->result);
         free (f->fatal_errnum_string);
         now_context_destroy (f->now);
         then_context_destroy (f->then);
         zlist_destroy (&f->queue);
         free (f);
+        errno = saved_errno;
     }
-    errno = saved_errno;
 }
 
 /* Create a future.
@@ -316,12 +317,24 @@ flux_future_t *flux_future_create (flux_future_init_f cb, void *arg)
     f->init = cb;
     f->init_arg = arg;
     f->queue = NULL;
+    f->refcount = 1;
     return f;
 error:
     flux_future_destroy (f);
     return NULL;
 }
 
+void flux_future_incref (flux_future_t *f)
+{
+    if (f)
+        f->refcount++;
+}
+
+void flux_future_decref (flux_future_t *f)
+{
+    flux_future_destroy (f);
+}
+
 static void post_fulfill (flux_future_t *f)
 {
     now_context_clear_timer (f->now);

src/common/libflux/future.h

@@ -66,6 +66,9 @@ flux_t *flux_future_get_flux (flux_future_t *f);
 void flux_future_set_reactor (flux_future_t *f, flux_reactor_t *r);
 flux_reactor_t *flux_future_get_reactor (flux_future_t *f);
 
+void flux_future_incref (flux_future_t *f);
+void flux_future_decref (flux_future_t *f);
+
 /* Composite future implementation
  */
 flux_future_t *flux_future_wait_all_create (void);