dlopen(), random, I/O, eval, and co-expressions (co-routines)

Makefile.am

@@ -5,14 +5,14 @@ LIBJQ_INCS = src/builtin.h src/bytecode.h src/compile.h                 \
         src/exec_stack.h src/jq_parser.h src/jv_alloc.h src/jv_dtoa.h   \
         src/jv_unicode.h src/jv_utf8_tables.h src/lexer.l src/libm.h    \
         src/linker.h src/locfile.h src/opcode_list.h src/parser.y       \
-        src/util.h
+        src/util.h src/builtin_modules.h
 
 LIBJQ_SRC = src/builtin.c src/bytecode.c src/compile.c src/execute.c    \
         src/jq_test.c src/jv.c src/jv_alloc.c src/jv_aux.c              \
         src/jv_dtoa.c src/jv_file.c src/jv_parse.c src/jv_print.c       \
         src/jv_unicode.c src/linker.c src/locfile.c src/util.c          \
         src/decNumber/decContext.c src/decNumber/decNumber.c            \
-        src/jv_dtoa_tsd.c                                               \
+        src/jv_dtoa_tsd.c src/builtin_modules.c                         \
         ${LIBJQ_INCS}
 
 ### C build options
@@ -57,11 +57,11 @@ libjq_la_LIBADD = -lm
 libjq_la_LDFLAGS = $(onig_LDFLAGS) -export-symbols-regex '^j[qv]_' -version-info 1:4:0
 
 if WIN32
-libjq_la_LIBADD += -lshlwapi
+libjq_la_LIBADD += -lshlwapi -lbcrypt
 libjq_la_LDFLAGS += -no-undefined
 endif
 
-include_HEADERS = src/jv.h src/jq.h
+include_HEADERS = src/jv.h src/jq.h src/jq_plugin.h
 
 if ENABLE_UBSAN
 AM_CFLAGS += -fsanitize=undefined
@@ -137,6 +137,11 @@ TESTS = tests/optionaltest tests/jqtest tests/onigtest tests/shtest tests/utf8te
 endif
 TESTS_ENVIRONMENT = NO_VALGRIND=$(NO_VALGRIND)
 
+lib_LTLIBRARIES += somod.la
+somod_la_SOURCES = tests/modules/somod/somod.c
+somod_la_CFLAGS = -DJQ_PLUGIN
+somod_la_LIBADD =
+somod_la_LDFLAGS = -module
 
 ### Building the manpage
 
@@ -185,6 +190,7 @@ EXTRA_DIST = $(DOC_FILES) $(man_MANS) $(TESTS) $(TEST_LOG_COMPILER)     \
         tests/modules/lib/jq/e/e.jq tests/modules/lib/jq/f.jq           \
         tests/modules/shadow1.jq tests/modules/shadow2.jq               \
         tests/modules/syntaxerror/syntaxerror.jq                        \
+        tests/modules/somod/somod.jq tests/modules/somod/somod.c        \
         tests/modules/test_bind_order.jq                                \
         tests/modules/test_bind_order0.jq                               \
         tests/modules/test_bind_order1.jq                               \

README.plugins.md

@@ -0,0 +1,107 @@
+jq FFI
+======
+
+jq now has an FFI.  You can create a module with C-coded builtins.
+
+A jq module's jq source can say:
+
+    module {"cfunctions":true};
+
+to cause a shared object or DLL of the same name to be loaded.
+
+An alternative shared object or DLL name can be specified instead:
+
+    module {"cfunctions":"my_shared_object"};
+
+The shared object or DLL must export a function named `jq_plugin_init`,
+or else it must list an alternative name for that function it its module
+meta:
+
+    module {"cfunctions":true,"plugin_init_function":"my_plugin_init"};
+
+The module's C sources must `#define JQ_PLUGIN` and must `#include` all
+of `<jv.h>`, `<jq.h>`, and `<jq_plugin.h>`.
+
+The init function has the following prototype:
+
+    typedef int (*jq_plugin_init_f)
+        (int,                       /* jq plugin min supported ABI version */
+         int,                       /* jq plugin max supported ABI version */
+         struct jq_state *,
+         const char **,             /* error string */
+         const char **,             /* jq-coded module contents */
+         size_t *,                  /* jq-coded module contents size */
+         struct cfunction **,       /* array of C-coded function descriptors */
+         size_t *);
+
+and must return 0 on success.  On failure, it should output an error
+string via the fourth argument.
+
+The plugin must check that two jq ABI integers include the
+`JQ_CURRENT_ABI` that the plugin was compiled against.
+
+The plugin can output a replacement for its jq-coded part via the fifth
+and sixth arguments.  If the fifth argument is not `NULL` and the sixth
+is `0`, then the string output in the fifth argument must be a C string
+(i.e., `NUL`-terminated).
+
+The plugin can output C-coded functions via the seventh and eigth init
+function arguments.
+
+Each C-coded function must have one the following prototypes:
+
+    jv my_func(struct jq_state *jq, jv input);
+    jv my_func(struct jq_state *jq, jv input, jv arg);
+    jv my_func(struct jq_state *jq, jv input, jv a0, jv a1);
+    ..
+    jv my_func(struct jq_state *jq, jv input, jv a0, jv a1, jv a2, jv a3, jv a4, jv a5);
+
+and must have an entry in the `struct cfunction` array output by the
+module's init function.
+
+`struct cfunction` is:
+
+    typedef void (*cfunction_ptr)();
+    struct cfunction {
+      cfunction_ptr fptr;       /* function pointer */
+      const char* name;         /* jq function name */
+      int nargs;                /* number of arguments; must be at least 1 */
+      unsigned int pure:1;      /* 1 if pure, 0 if impure */
+      unsigned int exported:1;  /* 1 if exported, 0 if not */
+    };
+
+
+jq plugins and DLL hell
+=======================
+
+All `jv` and `jq` functions in a plugin are replaced with macros of this
+form:
+
+    #define jv_fname ((*(struct jq_plugin_vtable **)jq)->jv_fname)
+
+Here `jq` should be the `struct jq_state *` argument to the C-coded
+builtin functions and the module's init function.
+
+This means that plugins can only call `jv` and `jq` functions in contexts
+where a `jq` variable is visible (and of type `struct jq_state *jq`).
+
+Since the plugin's init function and all its C-coded functions receive a
+`struct jq_state *jq` argument, a `struct jq_state *jq` should always be
+available for calling `jv` and `jq` functions.
+
+This way, plugins do not need to be linked with `libjq`.  This is quite
+important, as it limits DLL hell.  Essentially plugins "link" with
+`libjq` dynamically at run-time without having to rely on the system's
+RTLD to provide that functionality.
+
+This means that one can use the same plugin object in applications that
+use different versions of `libjq`, provided that the jq ABIs of the two
+`libjq` versions are sufficiently compatible (meaning, the layout of
+`jv` is the same, both provide all the functions needed by the plugin,
+and so on).
+
+Depending on the platform and how `libjq` or its callers are built
+(e.g., with versioned symbols or direct binding) it should even be
+possible even to have two different versions of `libjq` loaded in the
+*same process* and loading the same plugins, and the plugins will always
+use the `libjq` functions of the version of `libjq` calling them.

configure.ac

@@ -87,6 +87,10 @@ dnl Enable building all static
 AC_ARG_ENABLE([all-static],
    AC_HELP_STRING([--enable-all-static], [link jq with static libraries only]))
 
+if test "x$enable_all_static" != xyes; then
+  AC_FIND_FUNC([dlopen], [dl], [#include <dlfcn.h>], [0, 0])
+fi
+
 AS_IF([test "x$enable_docs" = "xyes"],[
    AC_CHECK_PROGS(pipenv_cmd, pipenv)
 
@@ -129,6 +133,8 @@ AM_CONDITIONAL([ENABLE_ALL_STATIC], [test "x$enable_all_static" = xyes])
 
 AC_FUNC_ALLOCA
 
+AC_FIND_FUNC([getline], [c], [#define _POSIX_C_SOURCE 200809L
+#include <stdio.h>], [0, 0, 0])
 AC_FIND_FUNC([isatty], [c], [#include <unistd.h>], [0])
 AC_FIND_FUNC([_isatty], [c], [#include <io.h>], [0])
 AC_FIND_FUNC([strptime], [c], [#include <time.h>], [0, 0, 0])