breaking changes to zig build API and improved caching

 * in Zig build scripts, getOutputPath() is no longer a valid function
   to call, unless setOutputDir() was used, or within a custom make()
   function. Instead there is more convenient API to use which takes
   advantage of the caching system. Search this commit diff for
   `exe.run()` for an example.
 * Zig build by default enables caching. All build artifacts will go
   into zig-cache. If you want to access build artifacts in a convenient
   location, it is recommended to add an `install` step. Otherwise
   you can use the `run()` API mentioned above to execute programs
   directly from their location in the cache. Closes #330.
   `addSystemCommand` is available for programs not built with Zig
   build.
 * Please note that Zig does no cache evicting yet. You may have to
   manually delete zig-cache directories periodically to keep disk
   usage down. It's planned for this to be a simple Least Recently
   Used eviction system eventually.
 * `--output`, `--output-lib`, and `--output-h` are removed. Instead,
   use `--output-dir` which defaults to the current working directory.
   Or take advantage of `--cache on`, which will print the main output
   path to stdout, and the other artifacts will be in the same directory
   with predictable file names. `--disable-gen-h` is available when
   one wants to prevent .h file generation.
 * `@cImport` is always independently cached now. Closes #2015.
   It always writes the generated Zig code to disk which makes debug
   info and compile errors better. No more "TODO: remember C source
   location to display here"
 * Fix .d file parsing. (Fixes the MacOS CI failure)
 * Zig no longer creates "temporary files" other than inside a
   zig-cache directory.

This breaks the CLI API that Godbolt uses. The suggested new invocation
can be found in this commit diff, in the changes to `test/cli.zig`.

build.zig

@@ -20,8 +20,8 @@ pub fn build(b: *Builder) !void {
         b.allocator,
         [][]const u8{ b.cache_root, "langref.html" },
     ) catch unreachable;
-    var docgen_cmd = b.addCommand(null, b.env_map, [][]const u8{
-        docgen_exe.getOutputPath(),
+    var docgen_cmd = docgen_exe.run();
+    docgen_cmd.addArgs([][]const u8{
         rel_zig_exe,
         "doc" ++ os.path.sep_str ++ "langref.html.in",
         langref_out_path,

doc/langref.html.in

@@ -7914,8 +7914,7 @@ pub fn build(b: *Builder) void {
 
     b.default_step.dependOn(&exe.step);
 
-    const run_cmd = b.addCommand(".", b.env_map, [][]const u8{exe.getOutputPath()});
-    run_cmd.step.dependOn(&exe.step);
+    const run_cmd = exe.run();
 
     const test_step = b.step("test", "Test the program");
     test_step.dependOn(&run_cmd.step);

example/mix_o_files/build.zig

@@ -4,14 +4,13 @@ pub fn build(b: *Builder) void {
     const obj = b.addObject("base64", "base64.zig");
 
     const exe = b.addExecutable("test", null);
-    exe.addCSourceFile("test.c",[][]const u8{"-std=c99"});
+    exe.addCSourceFile("test.c", [][]const u8{"-std=c99"});
     exe.addObject(obj);
     exe.linkSystemLibrary("c");
 
     b.default_step.dependOn(&exe.step);
 
-    const run_cmd = b.addCommand(".", b.env_map, [][]const u8{exe.getOutputPath()});
-    run_cmd.step.dependOn(&exe.step);
+    const run_cmd = exe.run();
 
     const test_step = b.step("test", "Test the program");
     test_step.dependOn(&run_cmd.step);

example/shared_library/build.zig

@@ -10,8 +10,7 @@ pub fn build(b: *Builder) void {
 
     b.default_step.dependOn(&exe.step);
 
-    const run_cmd = b.addCommand(".", b.env_map, [][]const u8{exe.getOutputPath()});
-    run_cmd.step.dependOn(&exe.step);
+    const run_cmd = exe.run();
 
     const test_step = b.step("test", "Test the program");
     test_step.dependOn(&run_cmd.step);

src/all_types.hpp

@@ -1087,7 +1087,6 @@ struct RootStruct {
     Buf *path; // relative to root_package->root_src_dir
     ZigList<size_t> *line_offsets;
     Buf *source_code;
-    AstNode *c_import_node;
     ZigLLVMDIFile *di_file;
 };
 
@@ -1746,13 +1745,12 @@ struct CodeGen {
 
     Buf triple_str;
     Buf global_asm;
-    Buf *out_h_path;
-    Buf *out_lib_path;
-    Buf artifact_dir;
     Buf output_file_path;
     Buf o_file_output_path;
-    Buf *wanted_output_file_path;
     Buf *cache_dir;
+    // As an input parameter, mutually exclusive with enable_cache. But it gets
+    // populated in codegen_build_and_link.
+    Buf *output_dir;
     Buf **libc_include_dir_list;
     size_t libc_include_dir_len;
 
@@ -1804,7 +1802,7 @@ struct CodeGen {
     bool verbose_cc;
     bool error_during_imports;
     bool generate_error_name_table;
-    bool enable_cache;
+    bool enable_cache; // mutually exclusive with output_dir
     bool enable_time_report;
     bool system_linker_hack;
     bool reported_bad_link_libc_error;
@@ -1844,6 +1842,7 @@ struct CodeGen {
     bool each_lib_rpath;
     bool disable_pic;
     bool is_dummy_so;
+    bool disable_gen_h;
 
     Buf *mmacosx_version_min;
     Buf *mios_version_min;

src/analyze.cpp

@@ -35,20 +35,6 @@ static bool is_top_level_struct(ZigType *import) {
 static ErrorMsg *add_error_note_token(CodeGen *g, ErrorMsg *parent_msg, ZigType *owner, Token *token, Buf *msg) {
     assert(is_top_level_struct(owner));
     RootStruct *root_struct = owner->data.structure.root_struct;
-    if (root_struct->c_import_node != nullptr) {
-        // if this happens, then translate_c generated code that
-        // failed semantic analysis, which isn't supposed to happen
-
-        Buf *note_path = buf_create_from_str("?.c");
-        Buf *note_source = buf_create_from_str("TODO: remember C source location to display here ");
-        ZigList<size_t> note_line_offsets = {0};
-        note_line_offsets.append(0);
-        ErrorMsg *note = err_msg_create_with_line(note_path, 0, 0,
-                note_source, &note_line_offsets, msg);
-
-        err_msg_add_note(parent_msg, note);
-        return note;
-    }
 
     ErrorMsg *err = err_msg_create_with_line(root_struct->path, token->start_line, token->start_column,
             root_struct->source_code, root_struct->line_offsets, msg);
@@ -60,17 +46,6 @@ static ErrorMsg *add_error_note_token(CodeGen *g, ErrorMsg *parent_msg, ZigType
 ErrorMsg *add_token_error(CodeGen *g, ZigType *owner, Token *token, Buf *msg) {
     assert(is_top_level_struct(owner));
     RootStruct *root_struct = owner->data.structure.root_struct;
-    if (root_struct->c_import_node != nullptr) {
-        // if this happens, then translate_c generated code that
-        // failed semantic analysis, which isn't supposed to happen
-        ErrorMsg *err = add_node_error(g, root_struct->c_import_node,
-            buf_sprintf("compiler bug: @cImport generated invalid zig code"));
-
-        add_error_note_token(g, err, owner, token, msg);
-
-        g->errors.append(err);
-        return err;
-    }
     ErrorMsg *err = err_msg_create_with_line(root_struct->path, token->start_line, token->start_column,
             root_struct->source_code, root_struct->line_offsets, msg);
 
@@ -1300,7 +1275,7 @@ static ZigTypeId container_to_type(ContainerKind kind) {
 }
 
 // This is like get_partial_container_type except it's for the implicit root struct of files.
-ZigType *get_root_container_type(CodeGen *g, const char *full_name, Buf *bare_name,
+static ZigType *get_root_container_type(CodeGen *g, const char *full_name, Buf *bare_name,
         RootStruct *root_struct)
 {
     ZigType *entry = new_type_table_entry(ZigTypeIdStruct);
@@ -4503,11 +4478,11 @@ ZigType *add_source_file(CodeGen *g, ZigPackage *package, Buf *resolved_path, Bu
     Buf *pkg_root_src_dir = &package->root_src_dir;
     Buf resolved_root_src_dir = os_path_resolve(&pkg_root_src_dir, 1);
 
-    assert(buf_starts_with_buf(resolved_path, &resolved_root_src_dir));
-
     Buf namespace_name = BUF_INIT;
     buf_init_from_buf(&namespace_name, &package->pkg_path);
     if (source_kind == SourceKindNonRoot) {
+        assert(buf_starts_with_buf(resolved_path, &resolved_root_src_dir));
+
         if (buf_len(&namespace_name) != 0) buf_append_char(&namespace_name, NAMESPACE_SEP_CHAR);
         buf_append_mem(&namespace_name, buf_ptr(&noextname) + buf_len(&resolved_root_src_dir) + 1,
             buf_len(&noextname) - (buf_len(&resolved_root_src_dir) + 1));

src/analyze.hpp

@@ -31,8 +31,6 @@ ZigType *get_array_type(CodeGen *g, ZigType *child_type, uint64_t array_size);
 ZigType *get_slice_type(CodeGen *g, ZigType *ptr_type);
 ZigType *get_partial_container_type(CodeGen *g, Scope *scope, ContainerKind kind,
         AstNode *decl_node, const char *full_name, Buf *bare_name, ContainerLayout layout);
-ZigType *get_root_container_type(CodeGen *g, const char *full_name, Buf *bare_name,
-        RootStruct *root_struct);
 ZigType *get_smallest_unsigned_int_type(CodeGen *g, uint64_t x);
 ZigType *get_error_union_type(CodeGen *g, ZigType *err_set_type, ZigType *payload_type);
 ZigType *get_bound_fn_type(CodeGen *g, ZigFn *fn_entry);
@@ -53,6 +51,7 @@ enum SourceKind {
     SourceKindRoot,
     SourceKindPkgMain,
     SourceKindNonRoot,
+    SourceKindCImport,
 };
 ZigType *add_source_file(CodeGen *g, ZigPackage *package, Buf *abs_full_path, Buf *source_code,
         SourceKind source_kind);
@@ -242,4 +241,6 @@ Error ensure_const_val_repr(IrAnalyze *ira, CodeGen *codegen, AstNode *source_no
 void typecheck_panic_fn(CodeGen *g, TldFn *tld_fn, ZigFn *panic_fn);
 Buf *type_bare_name(ZigType *t);
 Buf *type_h_name(ZigType *t);
+Error create_c_object_cache(CodeGen *g, CacheHash **out_cache_hash, bool verbose);
+
 #endif

src/ast_render.cpp

@@ -470,6 +470,9 @@ static void render_node_extra(AstRender *ar, AstNode *node, bool grouped) {
                         fprintf(ar->f, ", ");
                     }
                 }
+                if (node->data.fn_proto.is_var_args) {
+                    fprintf(ar->f, ", ...");
+                }
                 fprintf(ar->f, ")");
                 if (node->data.fn_proto.align_expr) {
                     fprintf(ar->f, " align(");

src/cache_hash.cpp

@@ -19,6 +19,8 @@ void cache_init(CacheHash *ch, Buf *manifest_dir) {
     ch->manifest_dir = manifest_dir;
     ch->manifest_file_path = nullptr;
     ch->manifest_dirty = false;
+    ch->force_check_manifest = false;
+    ch->b64_digest = BUF_INIT;
 }
 
 void cache_str(CacheHash *ch, const char *ptr) {
@@ -243,22 +245,21 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
     int rc = blake2b_final(&ch->blake, bin_digest, 48);
     assert(rc == 0);
 
-    if (ch->files.length == 0) {
+    buf_resize(&ch->b64_digest, 64);
+    base64_encode(buf_to_slice(&ch->b64_digest), {bin_digest, 48});
+
+    if (ch->files.length == 0 && !ch->force_check_manifest) {
         buf_resize(out_digest, 64);
         base64_encode(buf_to_slice(out_digest), {bin_digest, 48});
         return ErrorNone;
     }
 
-    Buf b64_digest = BUF_INIT;
-    buf_resize(&b64_digest, 64);
-    base64_encode(buf_to_slice(&b64_digest), {bin_digest, 48});
-
     rc = blake2b_init(&ch->blake, 48);
     assert(rc == 0);
     blake2b_update(&ch->blake, bin_digest, 48);
 
     ch->manifest_file_path = buf_alloc();
-    os_path_join(ch->manifest_dir, &b64_digest, ch->manifest_file_path);
+    os_path_join(ch->manifest_dir, &ch->b64_digest, ch->manifest_file_path);
 
     buf_append_str(ch->manifest_file_path, ".txt");
 
@@ -380,7 +381,7 @@ Error cache_hit(CacheHash *ch, Buf *out_digest) {
             blake2b_update(&ch->blake, chf->bin_digest, 48);
         }
     }
-    if (file_i < input_file_count) {
+    if (file_i < input_file_count || file_i == 0) {
         // manifest file is empty or missing entries, so this is a cache miss
         ch->manifest_dirty = true;
         for (; file_i < input_file_count; file_i += 1) {
@@ -442,6 +443,7 @@ Error cache_add_dep_file(CacheHash *ch, Buf *dep_file_path, bool verbose) {
         }
         if (opt_line.value.len == 0)
             continue;
+
         if (opt_line.value.ptr[0] == '"') {
             if (opt_line.value.len < 2) {
                 if (verbose) {
@@ -460,21 +462,28 @@ Error cache_add_dep_file(CacheHash *ch, Buf *dep_file_path, bool verbose) {
                 }
                 return ErrorInvalidDepFile;
             }
-        } else {
-            if (opt_line.value.ptr[opt_line.value.len - 1] == '\\') {
-                opt_line.value.len -= 2; // cut off ` \`
+            Buf *filename_buf = buf_create_from_slice(opt_line.value);
+            if ((err = cache_add_file(ch, filename_buf))) {
+                if (verbose) {
+                    fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(filename_buf), err_str(err));
+                    fprintf(stderr, "when processing .d file: %s\n", buf_ptr(dep_file_path));
+                }
+                return err;
             }
-            if (opt_line.value.len == 0)
-                continue;
-        }
-
-        Buf *filename_buf = buf_create_from_slice(opt_line.value);
-        if ((err = cache_add_file(ch, filename_buf))) {
-            if (verbose) {
-                fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(filename_buf), err_str(err));
-                fprintf(stderr, "when processing .d file: %s\n", buf_ptr(dep_file_path));
+        } else {
+            // sometimes there are multiple files on the same line; we actually need space tokenization.
+            SplitIterator line_it = memSplit(opt_line.value, str(" \t\\"));
+            Slice<uint8_t> filename;
+            while (SplitIterator_next(&line_it).unwrap(&filename)) {
+                Buf *filename_buf = buf_create_from_slice(filename);
+                if ((err = cache_add_file(ch, filename_buf))) {
+                    if (verbose) {
+                        fprintf(stderr, "unable to add %s to cache: %s\n", buf_ptr(filename_buf), err_str(err));
+                        fprintf(stderr, "when processing .d file: %s\n", buf_ptr(dep_file_path));
+                    }
+                    return err;
+                }
             }
-            return err;
         }
     }
     return ErrorNone;

src/cache_hash.hpp

@@ -25,8 +25,10 @@ struct CacheHash {
     ZigList<CacheHashFile> files;
     Buf *manifest_dir;
     Buf *manifest_file_path;
+    Buf b64_digest;
     OsFile manifest_file;
     bool manifest_dirty;
+    bool force_check_manifest;
 };
 
 // Always call this first to set up.
@@ -51,6 +53,10 @@ void cache_file_opt(CacheHash *ch, Buf *path);
 // If you got a cache hit, the next step is cache_release.
 // From this point on, there is a lock on the input params. Release
 // the lock with cache_release.
+// Set force_check_manifest if you plan to add files later, but have not
+// added any files before calling cache_hit. CacheHash::b64_digest becomes
+// available for use after this call, even in the case of a miss, and it
+// is a hash of the input parameters only.
 Error ATTRIBUTE_MUST_USE cache_hit(CacheHash *ch, Buf *out_b64_digest);
 
 // If you did not get a cache hit, call this function for every file