WIP: add dlopen support

Builds, and works, now to:

 - write tests
 - add support for builtin modules so we can `include "jq/prelude"` and so on

I've tested this with the following in a file named jqmod.jq:

module {"cfunctions":"jqmod"};
def add1: _add1;

and this in a file named jqmod.c built like so:

$ cc -shared -o jqmod.so -fPIC -O0 -ggdb3 jqmod.c -L $PWD/.libs -ljq -Wl,-rpath,$PWD/.libs
$ cat jqmod.c
\#include <stddef.h>
\#include <stdlib.h>
\#include <stdint.h>
\#include "src/jv.h"
\#include "src/jq.h"
\#include "src/jq_plugin.h"

\#define JQ_ABI 10700 /* 1.7.0 */

static jv f_add1(jq_state *jq, jv input) {
  if (jv_get_kind(input) == JV_KIND_NUMBER)
    return jv_number(jv_number_value(input) + 1);
  if (jv_get_kind(input) == JV_KIND_STRING)
    return jv_string_concat(input, jv_string("1"));
  if (jv_get_kind(input) == JV_KIND_ARRAY)
    return jv_array_append(input, jv_number(1));
  return jv_invalid_with_msg(jv_string("add1 only works for numbers, strings, and arrays"));
}

struct cfunction my_cfuncs[] = {
  { (cfunction_ptr)f_add1, "_add1", 1 },
};

int jq_plugin_init(int abi,
                   const char **err,
                   const char **contents,
                   size_t *contentssz,
                   struct cfunction **cf,
                   size_t *ncf)

{
  *contents = *err = 0;
  *ncf = *contentssz = 0;
  *cf = 0;

  if (abi != JQ_ABI) {
    *err = "Wrong ABI version";
    return 1;
  }

  *cf = my_cfuncs;
  *ncf = sizeof(my_cfuncs) / sizeof(my_cfuncs[0]);
  return 0;
}

Makefile.am

@@ -59,7 +59,7 @@ libjq_la_LIBADD += -lshlwapi
 libjq_la_LDFLAGS += -no-undefined
 endif
 
-include_HEADERS = src/jv.h src/jq.h
+include_HEADERS = src/jv.h src/jq.h src/jq_plugin.h
 
 if ENABLE_UBSAN
 AM_CFLAGS += -fsanitize=undefined

configure.ac

@@ -86,6 +86,10 @@ dnl Enable building all static
 AC_ARG_ENABLE([all-static],
    AC_HELP_STRING([--enable-all-static], [link jq with static libraries only]))
 
+if test "x$enable_all_static" != xyes; then
+  AC_FIND_FUNC([dlopen], [dl], [#include <dlfcn.h>], [0, 0])
+fi
+
 AS_IF([test "x$enable_docs" != "xno"],[
    AC_CHECK_PROGS(bundle_cmd, bundle)
 

src/bytecode.h

@@ -3,6 +3,7 @@
 #include <stdint.h>
 
 #include "jv.h"
+#include "jq_plugin.h"
 
 typedef enum {
 #define OP(name, imm, in, out) name,
@@ -43,15 +44,6 @@ struct opcode_description {
 
 const struct opcode_description* opcode_describe(opcode op);
 
-
-#define MAX_CFUNCTION_ARGS 10
-typedef void (*cfunction_ptr)();
-struct cfunction {
-  cfunction_ptr fptr;
-  const char* name;
-  int nargs;
-};
-
 struct symbol_table {
   struct cfunction* cfunctions;
   int ncfunctions;

src/jq_plugin.h

@@ -0,0 +1,27 @@
+
+#ifndef JQ_PLUGIN_H
+#define JQ_PLUGIN_H
+
+#include <stddef.h>
+#include <stdlib.h>
+#include <stdint.h>
+
+#define MAX_CFUNCTION_ARGS 10
+typedef void (*cfunction_ptr)();
+struct cfunction {
+  cfunction_ptr fptr;
+  const char* name;
+  int nargs;
+};
+
+typedef int (*jq_plugin_init_f)
+                  (int,                 /* jq plugin ABI version */
+                   const char **,       /* error string */
+                   const char **,       /* jq-coded module contents */
+                   size_t *,            /* jq-coded module contents size */
+                   struct cfunction **, /* array of C-coded function descriptors */
+                   size_t *);
+
+#define JQ_ABI 10700 /* 1.7.0 */
+
+#endif /* JQ_PLUGIN_H */

src/linker.c

@@ -19,13 +19,61 @@
 #include "parser.h"
 #include "util.h"
 #include "compile.h"
+#include "bytecode.h" // For struct cfunction, which will have to move elsewhere
 #include "jv_alloc.h"
 
+#ifdef WIN32
+#include <windows.h>
+#include <libloaderapi.h>
+
+typedef HMODULE jq_dl_handle;
+#define JQ_DL_OPEN(name) LoadLibrary(name)
+#define JQ_DL_CLOSE(h) FreeLibrary(h)
+#define JQ_DL_SYM(h, sym) GetProcAddress(h, sym)
+#define JQ_PATH_SEP "\\"
+#define JQ_DLL_EXT ".dll"
+
+#elif defined(HAVE_DLOPEN)
+#include <dlfcn.h>
+
+#ifndef RTLD_GROUP
+#define RTLD_GROUP 0
+#endif
+
+typedef void *jq_dl_handle;
+#define JQ_DL_OPEN(name) dlopen(name, RTLD_LAZY | RTLD_LOCAL | RTLD_GROUP)
+#define JQ_DL_CLOSE(h) dlclose(h)
+#define JQ_DL_SYM(h, sym) dlsym(h, sym)
+#define JQ_PATH_SEP "/"
+#define JQ_DLL_EXT ".so"
+
+#else
+
+typedef void *jq_dl_handle;
+#define JQ_DL_OPEN(name) (NULL)
+#define JQ_DL_CLOSE(h)
+#define JQ_DL_SYM(h, sym) (NULL)
+#define JQ_PATH_SEP "/"
+#define JQ_DLL_EXT ".so"
+
+#endif
+
+struct jq_lib {
+  jv jname;
+  jv jbasename; /* basename() of jname */
+  jv origin;    /* dirname()  of jname */
+  jv contents;  /* jq code or JSON text */
+  const char *name;   /* jv_string_value(jname) */
+  block defs;   /* parsed library program */
+  jq_dl_handle h;
+  struct cfunction *cfuncs;
+  size_t ncfuncs;
+};
 struct lib_loading_state {
-  char **names;
-  block *defs;
-  uint64_t ct;
+  struct jq_lib *libs;
+  size_t nlibs;
 };
+
 static int load_library(jq_state *jq, jv lib_path, int is_data, int raw,
                         const char *as, block *out_block,
                         struct lib_loading_state *lib_state);
@@ -287,15 +335,15 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block
       }
     } else {
       uint64_t state_idx = 0;
-      for (; state_idx < lib_state->ct; ++state_idx) {
-        if (strcmp(lib_state->names[state_idx],jv_string_value(resolved)) == 0)
+      for (; state_idx < lib_state->nlibs; ++state_idx) {
+        if (strcmp(lib_state->libs[state_idx].name,jv_string_value(resolved)) == 0)
           break;
       }
 
-      if (state_idx < lib_state->ct) { // Found
+      if (state_idx < lib_state->nlibs) { // Found
         jv_free(resolved);
         // Bind the library to the program
-        bk = block_bind_library(lib_state->defs[state_idx], bk, OP_IS_CALL_PSEUDO, as_str);
+        bk = block_bind_library(lib_state->libs[state_idx].defs, bk, OP_IS_CALL_PSEUDO, as_str);
       } else { // Not found.   Add it to the table before binding.
         block dep_def_block = gen_noop();
         nerrors += load_library(jq, resolved, is_data, raw, as_str, &dep_def_block, lib_state);
@@ -315,53 +363,145 @@ static int process_dependencies(jq_state *jq, jv jq_origin, jv lib_origin, block
   return nerrors;
 }
 
+static int load_shared_object(jq_state *jq, struct jq_lib *lib) {
+  jq_plugin_init_f get_cfuncs;
+  int ret = 0;
+  const char *err = 0;
+  const char *data = 0;
+  size_t datasz = 0;
+  jv meta = block_module_meta(lib->defs);
+  jv oname = jv_invalid();
+  jv tmp = jv_invalid();
+
+  if (jv_get_kind(meta) != JV_KIND_OBJECT ||
+      !jv_is_valid((oname = jv_object_get(jv_copy(meta), jv_string("cfunctions")))))
+    goto out;
+
+  ret = 1;
+  if (jv_get_kind(oname) != JV_KIND_STRING) {
+    jq_report_error(jq, jv_string_fmt("module %s: value of \"cfunctions\" key must be a string",
+                                      lib->name));
+    goto out;
+  }
+  oname = jv_string_concat(oname, jv_string(JQ_DLL_EXT));
+  if (!jv_is_valid((tmp = validate_relpath(jv_copy(oname))))) {
+    jq_report_error(jq, jv_string_fmt("module %s: path for DLL / shared object is not relative: %s: %s",
+                                      lib->name, jv_string_value(oname), jv_string_value(tmp)));
+    goto out;
+  }
+  /* A more idiomatic version of this would be nice: */
+  jv_free(tmp);
+  tmp = jv_string_fmt("%s%s%s", jv_string_value(lib->origin), JQ_PATH_SEP, jv_string_value(oname));
+  jv_free(oname);
+  oname = tmp;
+  tmp = jv_invalid();
+
+  if ((lib->h = JQ_DL_OPEN(jv_string_value(oname))) == NULL) {
+    jq_report_error(jq, jv_string_fmt("module %s: could not load DLL / shared object: %s",
+                                      lib->name, jv_string_value(oname)));
+    goto out;
+  }
+  if ((get_cfuncs = JQ_DL_SYM(lib->h, "jq_plugin_init")) == NULL) {
+    jq_report_error(jq, jv_string_fmt("module %s: could find \"jq_plugin_init\" symbol in DLL / shared object: %s",
+                                      lib->name, jv_string_value(oname)));
+    goto out;
+  }
+
+  if ((ret = get_cfuncs(JQ_ABI, &err, &data, &datasz, &lib->cfuncs, &lib->ncfuncs))) {
+    jq_report_error(jq, jv_string_fmt("module %s: failed to initialize DLL / shared object: %s",
+                                      lib->name, err ? err : "<unknown error>"));
+    goto out;
+  }
+
+  ret = 0;
+  if (data && datasz && datasz < INT_MAX) {
+    struct locfile* src = NULL;
+
+    jv_free(lib->contents);
+    block_free(lib->defs);
+    lib->contents = jv_string_sized(data, datasz);
+
+    src = locfile_init(jq, lib->name,
+                       jv_string_value(lib->contents),
+                       jv_string_length_bytes(jv_copy(lib->contents)));
+    ret = jq_parse_library(src, &lib->defs);
+    locfile_free(src);
+  }
+
+out:
+  jv_free(oname);
+  jv_free(meta);
+  jv_free(tmp);
+  return ret;
+}
+
 // Loads the library at lib_path into lib_state, putting the library's defs
 // into *out_block
 static int load_library(jq_state *jq, jv lib_path, int is_data, int raw, const char *as, block *out_block, struct lib_loading_state *lib_state) {
   int nerrors = 0;
   struct locfile* src = NULL;
-  block program;
-  jv data;
+  char *dname = strdup(jv_string_value(lib_path));
+  char *bname = strdup(dname);
+  size_t state_idx = lib_state->nlibs++;
+
+  lib_state->libs = jv_mem_realloc(lib_state->libs, lib_state->nlibs * sizeof(lib_state->libs[0]));
+  struct jq_lib *this_lib = &lib_state->libs[state_idx];
+
+  this_lib->jname = jv_copy(lib_path);
+  this_lib->origin = dname ? jv_string(dirname(dname)) : jv_invalid_with_msg(jv_false()); // XXX ENOMEM
+  this_lib->jbasename = bname ? jv_string(basename(bname)) : jv_invalid_with_msg(jv_false()); // XXX ENOMEM
+  this_lib->name = jv_string_value(this_lib->jname);
+  this_lib->defs = gen_noop();
+  this_lib->h = NULL;
+  this_lib->cfuncs = NULL;
+  this_lib->ncfuncs = 0;
+
   if (is_data && !raw)
-    data = jv_load_file(jv_string_value(lib_path), 0);
+    this_lib->contents = jv_load_file(jv_string_value(lib_path), 0);
   else
-    data = jv_load_file(jv_string_value(lib_path), 1);
-  int state_idx;
-  if (!jv_is_valid(data)) {
-    if (jv_invalid_has_msg(jv_copy(data)))
-      data = jv_invalid_get_msg(data);
+    this_lib->contents = jv_load_file(jv_string_value(lib_path), 1);
+  free(dname);
+  free(bname);
+
+  if (!jv_is_valid(this_lib->contents)) {
+    if (jv_invalid_has_msg(jv_copy(this_lib->contents)))
+      this_lib->contents = jv_invalid_get_msg(this_lib->contents);
     else
-      data = jv_string("unknown error");
-    jq_report_error(jq, jv_string_fmt("jq: error loading data file %s: %s\n", jv_string_value(lib_path), jv_string_value(data)));
+      this_lib->contents = jv_string("unknown error");
+    jq_report_error(jq, jv_string_fmt("jq: error loading data file %s: %s\n",
+                                      jv_string_value(lib_path),
+                                      jv_string_value(this_lib->contents)));
     nerrors++;
     goto out;
   } else if (is_data) {
     // import "foo" as $bar;
-    program = gen_const_global(jv_copy(data), as);
+    this_lib->defs = gen_const_global(jv_copy(this_lib->contents), as);
   } else {
-    // import "foo" as bar;
-    src = locfile_init(jq, jv_string_value(lib_path), jv_string_value(data), jv_string_length_bytes(jv_copy(data)));
-    nerrors += jq_parse_library(src, &program);
+    // import "foo" [as bar] OR include "foo";
+    src = locfile_init(jq, jv_string_value(lib_path),
+                       jv_string_value(this_lib->contents),
+                       jv_string_length_bytes(jv_copy(this_lib->contents)));
+    nerrors += jq_parse_library(src, &this_lib->defs);
+    /*
+     * Load the shared object now because it may replace the jq code of the
+     * library.
+     */
+    if (nerrors == 0)
+      nerrors += load_shared_object(jq, this_lib);
     if (nerrors == 0) {
-      char *lib_origin = strdup(jv_string_value(lib_path));
       nerrors += process_dependencies(jq, jq_get_jq_origin(jq),
-                                      jv_string(dirname(lib_origin)),
-                                      &program, lib_state);
-      free(lib_origin);
-      program = block_bind_self(program, OP_IS_CALL_PSEUDO);
+                                      jv_copy(this_lib->origin), &this_lib->defs,
+                                      lib_state);
+      if (this_lib->ncfuncs)
+        this_lib->defs = gen_cbinding(this_lib->cfuncs, this_lib->ncfuncs, this_lib->defs);
+      this_lib->defs = block_bind_self(this_lib->defs, OP_IS_CALL_PSEUDO);
     }
   }
-  state_idx = lib_state->ct++;
-  lib_state->names = jv_mem_realloc(lib_state->names, lib_state->ct * sizeof(const char *));
-  lib_state->defs = jv_mem_realloc(lib_state->defs, lib_state->ct * sizeof(block));
-  lib_state->names[state_idx] = strdup(jv_string_value(lib_path));
-  lib_state->defs[state_idx] = program;
-  *out_block = program;
+  *out_block = this_lib->defs;
   if (src)
     locfile_free(src);
 out:
   jv_free(lib_path);
-  jv_free(data);
   return nerrors;
 }
 
@@ -395,7 +535,7 @@ jv load_module_meta(jq_state *jq, jv mod_relpath) {
 int load_program(jq_state *jq, struct locfile* src, block *out_block) {
   int nerrors = 0;
   block program;
-  struct lib_loading_state lib_state = {0,0,0};
+  struct lib_loading_state lib_state = {0,0};
   nerrors = jq_parse(src, &program);
   if (nerrors)
     return nerrors;
@@ -412,15 +552,28 @@ int load_program(jq_state *jq, struct locfile* src, block *out_block) {
 
   nerrors = process_dependencies(jq, jq_get_jq_origin(jq), jq_get_prog_origin(jq), &program, &lib_state);
   block libs = gen_noop();
-  for (uint64_t i = 0; i < lib_state.ct; ++i) {
-    free(lib_state.names[i]);
-    if (nerrors == 0 && !block_is_const(lib_state.defs[i]))
-      libs = block_join(libs, lib_state.defs[i]);
-    else
-      block_free(lib_state.defs[i]);
+  for (size_t i = 0; i < lib_state.nlibs; ++i) {
+    if (nerrors == 0 && !block_is_const(lib_state.libs[i].defs)) {
+      libs = block_join(libs, lib_state.libs[i].defs);
+      lib_state.libs[i].defs = gen_noop();
+    }
+#if 0
+    /*
+     * XXX Gaaa, we can't dlclose() here as we'll dlclose() to soon.  We must
+     * defer this until the jq_state is torn down.
+     *
+     * We might as well not even do any of the lib_state cleanup here and let
+     * jq_teardown() to it (by calling a library_free() function.
+     */
+    if (lib_state.libs[i].h)
+      JQ_DL_CLOSE(lib_state.libs[i].h);
+#endif
+    jv_free(lib_state.libs[i].jname);
+    jv_free(lib_state.libs[i].jbasename);
+    jv_free(lib_state.libs[i].origin);
+    jv_free(lib_state.libs[i].contents);
+    block_free(lib_state.libs[i].defs);
   }
-  free(lib_state.names);
-  free(lib_state.defs);
   if (nerrors)
     block_free(program);
   else